pax_global_header00006660000000000000000000000064152176421030014513gustar00rootroot0000000000000052 comment=557becfb64c503ae9c04344b0047661f43f44320 xgrammar-0.2.3/000077500000000000000000000000001521764210300133335ustar00rootroot00000000000000xgrammar-0.2.3/.clang-format000066400000000000000000000002571521764210300157120ustar00rootroot00000000000000BasedOnStyle: Google DerivePointerAlignment: false ColumnLimit: 100 PointerAlignment: Left AlignAfterOpenBracket: BlockIndent BinPackArguments: false BinPackParameters: false xgrammar-0.2.3/.cmake-format.yaml000066400000000000000000000001731521764210300166440ustar00rootroot00000000000000format: line_width: 100 tab_size: 2 dangle_parens: true command_case: lower keyword_case: upper autosort: true xgrammar-0.2.3/.github/000077500000000000000000000000001521764210300146735ustar00rootroot00000000000000xgrammar-0.2.3/.github/ISSUE_TEMPLATE/000077500000000000000000000000001521764210300170565ustar00rootroot00000000000000xgrammar-0.2.3/.github/ISSUE_TEMPLATE/bug_report.yml000066400000000000000000000021661521764210300217560ustar00rootroot00000000000000name: 🐛 Bug Report description: Create a report to help us reproduce and fix the bug title: "[Bug] " labels: ['Bug'] body: - type: checkboxes attributes: label: Checklist options: - label: 1. I have searched related issues but cannot get the expected help. - label: 2. The bug has not been fixed in the latest version. - label: 3. I have filled all the required information below. - label: 4. The bug report is in English. - type: textarea attributes: label: Describe the bug description: A clear and concise description of what the bug is. validations: required: true - type: textarea attributes: label: Reproduction description: | Please include a minimal code snippet to reproduce the behavior. placeholder: | A placeholder for the command. validations: required: true - type: textarea attributes: label: Environment description: | Please provide the version of XGrammar, the serving engine for llms, etc. placeholder: Environment here. validations: required: true xgrammar-0.2.3/.github/ISSUE_TEMPLATE/feature_request.yml000066400000000000000000000024251521764210300230070ustar00rootroot00000000000000name: 🚀 Feature Request description: Suggest an idea for this project title: "[Feature] " labels: ['Feature'] body: - type: checkboxes attributes: label: Checklist options: - label: 1. I have searched related issues but cannot get the expected help. - label: 2. The latest version does not have this feature. - label: 3. I have filled all the required information below. - label: 4. The feature request is in English. - type: textarea attributes: label: requested feature description: | Please describe what's the requested feature. placeholder: | A clear and concise description of the requested feature. validations: required: true - type: textarea attributes: label: Motivation description: | Please describe the motivation for the feature. placeholder: | A clear and concise description of the motivation for the feature. validations: required: true - type: textarea attributes: label: Alternatives description: | Please describe the alternatives you have considered. placeholder: | A clear and concise description of the alternatives you have considered, if any. validations: required: false xgrammar-0.2.3/.github/workflows/000077500000000000000000000000001521764210300167305ustar00rootroot00000000000000xgrammar-0.2.3/.github/workflows/benchmark.yaml000066400000000000000000000022151521764210300215460ustar00rootroot00000000000000name: XGrammar Benchmark on: workflow_dispatch: schedule: - cron: '0 0 * * *' jobs: run_benchmark: name: Run XGrammar Benchmark if: github.ref == 'refs/heads/main'&& github.repository_owner == 'mlc-ai' runs-on: [self-hosted, Linux, X64] steps: - name: Checkout uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python 3.11 uses: actions/setup-python@v5 with: python-version: 3.11 - name: Build xgrammar from source run: | python -m pip install --upgrade pip pip install . - name: Install dependencies run: | pip install torch transformers datasets tqdm requests - name: Run benchmark id: benchmark env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | python examples/benchmark/cibench_grammar_compile_mask_gen.py --num_iters 3 --num_warmup 2 --datasets all | tee benchmark_output.txt - name: Upload benchmark results uses: actions/upload-artifact@v4 with: name: benchmark-results path: benchmark_output.txt xgrammar-0.2.3/.github/workflows/build_and_release.yaml000066400000000000000000000062741521764210300232460ustar00rootroot00000000000000name: Build and upload to PyPI on: workflow_dispatch: pull_request: push: branches: - main release: types: - published jobs: build_wheels: name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: - {os: ubuntu-latest, arch: x86_64, build: 'cp*-manylinux*'} - {os: ubuntu-24.04-arm, arch: aarch64, build: 'cp*-manylinux*'} - {os: windows-latest, arch: AMD64, build: 'cp*'} - {os: macos-14, arch: arm64, build: 'cp*'} - {os: macos-15-intel, arch: x86_64, build: 'cp*',} steps: - uses: astral-sh/setup-uv@v4 - uses: actions/checkout@v4 with: submodules: recursive - name: Use LLVM clang 17 on macOS if: matrix.os == 'macos-15-intel' && matrix.arch == 'x86_64' run: | brew update brew install llvm@17 echo "CC=$(brew --prefix llvm@17)/bin/clang" >> "$GITHUB_ENV" echo "CXX=$(brew --prefix llvm@17)/bin/clang++" >> "$GITHUB_ENV" echo "$(brew --prefix llvm@17)/bin" >> "$GITHUB_PATH" clang++ --version - name: Build wheels uses: pypa/cibuildwheel@v3.3.0 env: CIBW_ARCHS_MACOS: ${{ matrix.arch }} CIBW_ARCHS_LINUX: ${{ matrix.arch }} CIBW_ARCHS_WINDOWS: ${{ matrix.arch }} CIBW_BUILD: ${{ matrix.build }} CIBW_TEST_SKIP: '*' CIBW_BUILD_VERBOSITY: 1 - uses: actions/upload-artifact@v4 with: name: cibw-wheels-${{ matrix.os }}-${{ matrix.arch }}-${{ strategy.job-index }} path: ./wheelhouse/*.whl build_sdist: name: Build source distribution runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: submodules: recursive - uses: astral-sh/setup-uv@v4 - name: Build sdist run: uv tool run --from build pyproject-build --sdist - name: Check metadata run: pipx run twine check dist/* - uses: actions/upload-artifact@v4 with: name: cibw-sdist path: dist/*.tar.gz upload_pypi: needs: [build_wheels, build_sdist] runs-on: ubuntu-latest environment: pypi permissions: id-token: write attestations: write if: github.event_name == 'release' && github.event.action == 'published' # or, alternatively, upload to PyPI on every tag starting with 'v' (remove on: release above to use this) # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') steps: - uses: actions/download-artifact@v4 with: # unpacks all CIBW artifacts into dist/ pattern: cibw-* path: dist merge-multiple: true - name: Generate artifact attestation for sdist and wheels uses: actions/attest-build-provenance@v1 with: subject-path: dist/* - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: attestations: true verbose: true # repository-url: https://test.pypi.org/legacy/ # To test: repository-url: https://test.pypi.org/legacy/ xgrammar-0.2.3/.github/workflows/build_web_xgrammar.yaml000066400000000000000000000015001521764210300234420ustar00rootroot00000000000000name: Build Web-XGrammar on: workflow_dispatch: pull_request: push: branches: - main jobs: build: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 with: submodules: recursive - name: Build emsdk run: | git clone https://github.com/emscripten-core/emsdk.git cd emsdk ./emsdk install latest ./emsdk activate latest - name: Build Web-XGrammar run: | source ./emsdk/emsdk_env.sh echo "set(XGRAMMAR_BUILD_PYTHON_BINDINGS OFF)" >> cmake/config.cmake cd web npm install npm run build # TODO(Linzhang): Tests will fail currently. Need further maintenance. # - name: Run tests # run: | # cd web # npm test xgrammar-0.2.3/.github/workflows/close_issues.yaml000066400000000000000000000017741521764210300223250ustar00rootroot00000000000000name: Close inactive issues on: workflow_dispatch: schedule: - cron: "0 0 * * *" jobs: close-issues: runs-on: ubuntu-latest permissions: issues: write pull-requests: write steps: - uses: actions/stale@v9 with: days-before-issue-stale: 60 days-before-issue-close: 14 stale-issue-label: "stale" exempt-issue-labels: "pinned,important" stale-issue-message: > This issue has been inactive for 60 days and is marked as stale. Please confirm if this issue is still relevant by commenting or removing the 'stale' label within 14 days, otherwise it will be closed automatically. close-issue-message: > Closing this issue due to inactivity for 14 days since it was marked as stale. Feel free to reopen if you believe it's still relevant. days-before-pr-stale: -1 days-before-pr-close: -1 repo-token: ${{ secrets.GITHUB_TOKEN }} xgrammar-0.2.3/.github/workflows/documentation.yaml000066400000000000000000000020751521764210300224710ustar00rootroot00000000000000name: Build Docs on: workflow_dispatch: pull_request: push: branches: - main jobs: deploy_docs: name: Deploy Docs runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.12' cache: 'pip' - name: Setup Ruby uses: ruby/setup-ruby@v1 with: ruby-version: '3.0' - name: Installing dependencies run: | python -m pip install -r docs/requirements.txt gem install jekyll jekyll-remote-theme - name: Deploying on GitHub Pages if: github.ref == 'refs/heads/main' && github.repository_owner == 'mlc-ai' run: | git remote set-url origin https://x-access-token:${{ secrets.MLC_GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY git config --global user.email "mlc-gh-actions-bot@nomail" git config --global user.name "mlc-gh-actions-bot" ./scripts/gh_deploy_site.sh xgrammar-0.2.3/.github/workflows/routine_unit_test.yaml000066400000000000000000000043561521764210300234070ustar00rootroot00000000000000name: Routine Unit Test on: workflow_dispatch: schedule: - cron: '0 0 * * *' jobs: run_unit_test: if: github.ref == 'refs/heads/main' name: Run unit tests on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-14, macos-15-intel] python: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] exclude: - os: macos-15-intel python: '3.13' # The reason for the exclusion is that pytorch distribution # can't be found by pip on macos-15-intel with python 3.13. steps: - name: Checkout uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - name: Cache huggingface uses: actions/cache@v4 with: path: | ~/.cache/huggingface key: huggingface-${{ matrix.os }}-python-${{ matrix.python }} - name: Use LLVM clang 17 on macOS if: matrix.os == 'macos-15-intel' run: | brew update brew install llvm@17 echo "CC=$(brew --prefix llvm@17)/bin/clang" >> "$GITHUB_ENV" echo "CXX=$(brew --prefix llvm@17)/bin/clang++" >> "$GITHUB_ENV" echo "$(brew --prefix llvm@17)/bin" >> "$GITHUB_PATH" clang++ --version - name: Build xgrammar from source run: | echo "set(XGRAMMAR_BUILD_CXX_TESTS ON)" >> cmake/config.cmake echo "set(XGRAMMAR_ENABLE_INTERNAL_CHECK ON)" >> cmake/config.cmake python -m pip install --upgrade pip pip install -v ".[test]" - name: Run C++ tests run: | ctest --test-dir build -V --timeout 30 --stop-on-failure - name: Run Python tests env: HF_TOKEN: ${{ secrets.HF_TOKEN }} HF_HUB_DOWNLOAD_TIMEOUT: 60 if: env.HF_TOKEN != '' run: | pytest - name: Run Python tests without HF_TOKEN env: HF_TOKEN: ${{ secrets.HF_TOKEN }} if: env.HF_TOKEN == '' run: | pytest -m "not hf_token_required" xgrammar-0.2.3/.github/workflows/tmate.yaml000066400000000000000000000017121521764210300207270ustar00rootroot00000000000000on: workflow_dispatch: jobs: run_tmate_session: name: Run tmate session on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-14, macos-15-intel] python: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] exclude: - os: macos-15-intel python: '3.13' # The reason for the exclusion is that pytorch distribution # can't be found by pip on macos-15-intel with python 3.13. steps: - name: Checkout uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - name: Setup tmate session uses: mxschmitt/action-tmate@v3 timeout-minutes: 30 with: limit-access-to-actor: true xgrammar-0.2.3/.github/workflows/unit_test.yaml000066400000000000000000000063341521764210300216400ustar00rootroot00000000000000on: workflow_dispatch: pull_request: push: branches: - main jobs: pre_check: name: Pre-check on Ubuntu-latest runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python uses: actions/setup-python@v5 - name: Pre-commit uses: pre-commit/action@v3.0.1 - name: Ruff check uses: astral-sh/ruff-action@v3 run_unit_test: needs: [pre_check] name: Run unit tests on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-14, macos-15-intel] python: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.14t'] exclude: - os: macos-14 python: '3.14t' - os: macos-15-intel python: '3.13' - os: macos-15-intel python: '3.14' - os: macos-15-intel python: '3.14t' # The reason for the exclusion is that pytorch distribution # can't be found by pip on macos-15-intel with python 3.13. steps: - name: Checkout uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - name: Cache huggingface uses: actions/cache@v4 with: path: | ~/.cache/huggingface key: huggingface-${{ matrix.os }}-python-${{ matrix.python }} - name: Use LLVM clang 17 on macOS if: matrix.os == 'macos-15-intel' run: | brew update brew install llvm@17 echo "CC=$(brew --prefix llvm@17)/bin/clang" >> "$GITHUB_ENV" echo "CXX=$(brew --prefix llvm@17)/bin/clang++" >> "$GITHUB_ENV" echo "$(brew --prefix llvm@17)/bin" >> "$GITHUB_PATH" clang++ --version - name: Build xgrammar from source run: | echo "set(XGRAMMAR_BUILD_CXX_TESTS ON)" >> cmake/config.cmake echo "set(XGRAMMAR_ENABLE_INTERNAL_CHECK ON)" >> cmake/config.cmake python -m pip install --upgrade pip pip install -v ".[test]" # Test with MLX (which is optional) under macOS arm64 - name: Install MLX if: matrix.os == 'macos-14' && !endsWith(matrix.python, 't') run: | pip install mlx-lm - name: Run C++ tests run: | ctest --test-dir build -V --timeout 30 --stop-on-failure - name: Run Python tests env: HF_TOKEN: ${{ secrets.HF_TOKEN }} HF_HUB_DOWNLOAD_TIMEOUT: 60 if: env.HF_TOKEN != '' run: | pytest - name: Run Python tests without HF_TOKEN env: HF_TOKEN: ${{ secrets.HF_TOKEN }} if: env.HF_TOKEN == '' run: | pytest -m "not hf_token_required" - name: Run tests under pytest-run-parallel env: PYTHON_GIL: 0 if: matrix.python == '3.14t' run: | python -m pip install pytest-run-parallel pytest -m "not hf_token_required" --parallel-threads=8 xgrammar-0.2.3/.gitignore000066400000000000000000000065171521764210300153340ustar00rootroot00000000000000/tmp/ *.bak # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class .DS_Store *.S # C extensions *.so build/ *.ll .npm # Distribution / packaging .Python env/ build/ build-*/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST .conda/ # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Generated by python/gen_requirements.py python/requirements/*.txt # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ /Testing/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ docs/_staging/ # PyBuilder target/ /target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject *~ *.pyc *~ config.mk /config.cmake Win32 *.dir perf *.wasm .emscripten ## IOS DerivedData/ ## Java *.class *.worksheet *.idea *.iml *.classpath *.project *.settings */node_modules/ ## Various settings *.pbxuser !default.pbxuser *.mode1v3 !default.mode1v3 *.mode2v3 !default.mode2v3 *.perspectivev3 !default.perspectivev3 xcuserdata/ .pkl_memoize_* .emscripten* .m2 # Compiled Dynamic libraries *.so *.dylib *.dll # Compiled Object files *.slo *.lo *.o *.obj # Precompiled Headers *.gch *.pch # Compiled Static libraries *.lai *.la *.a *.lib # Executables *.exe *.out *.app ## Other *.moved-aside *.xccheckout *.xcscmblueprint .DS_Store tags cscope* *.lock # vim temporary files *.swp *.swo .bash_history # *.json *.params *.ro *.onnx *.h5 # Mac OS X .DS_Store # Jetbrain .idea .ipython .jupyter .nv .pylint.d .python_history .pytest_cache .local cmake-build-debug # Visual Studio .vs # Visual Studio Code .vscode # tmp file .nfs* # keys *.pem *.p12 *.pfx *.cer *.crt *.der # patch sentinel patched.txt # Python type checking .mypy_cache/ .pyre/ # pipenv files Pipfile Pipfile.lock # conda package artifacts conda/Dockerfile.cuda* conda/pkg .node_repl_history # nix files .envrc *.nix # Docker files .sudo_as_admin_successful # Local docs build _docs/ .config/configstore/ .ci-py-scripts/ # Used in CI to communicate between Python and Jenkins .docker-image-names/ # GDB history file .gdb_history # AI assistant files .cursor/ # automatically generated by tvm-ffi-stubgen python/xgrammar/tvm_ffi_binding/ .build/ xgrammar-0.2.3/.gitmodules000066400000000000000000000004751521764210300155160ustar00rootroot00000000000000[submodule "3rdparty/dlpack"] path = 3rdparty/dlpack url = https://github.com/dmlc/dlpack.git [submodule "3rdparty/googletest"] path = 3rdparty/googletest url = https://github.com/google/googletest.git [submodule "3rdparty/cpptrace"] path = 3rdparty/cpptrace url = https://github.com/jeremy-rifkin/cpptrace.git xgrammar-0.2.3/.pre-commit-config.yaml000066400000000000000000000033361521764210300176210ustar00rootroot00000000000000# To run for staged files: # # pre-commit run # # To run for all files: # # pre-commit run -a # # To run every time you commit in git: # # pre-commit install # # To update this file: # # pre-commit autoupdate # # See https://github.com/pre-commit/pre-commit # Note the pre-commit hooks should only be used for formatting, but not for linting. # For linting consider using CI. repos: # Standard hooks - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: - id: check-added-large-files - id: check-case-conflict - id: check-merge-conflict - id: check-symlinks - id: end-of-file-fixer - id: mixed-line-ending - id: requirements-txt-fixer - id: trailing-whitespace # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks rev: v1.5.5 hooks: - id: remove-tabs - id: remove-crlf # Formatters - repo: https://github.com/psf/black-pre-commit-mirror rev: 24.1.0 hooks: - id: black - repo: https://github.com/pycqa/isort rev: 6.0.0 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-clang-format rev: v19.1.7 hooks: - id: clang-format types_or: [c++, c, cuda] exclude: | (?x)^(.*cubin.cpp$ | .*fmha_cubin.h | 3rdparty/.*)$ - repo: https://github.com/cheshirekow/cmake-format-precommit rev: v0.6.13 hooks: - id: cmake-format additional_dependencies: [pyyaml>=5.1] - repo: https://github.com/google/yamlfmt rev: v0.16.0 hooks: - id: yamlfmt - repo: https://github.com/ComPWA/taplo-pre-commit rev: v0.9.3 hooks: - id: taplo-format args: ["--option", "column_width=100"] xgrammar-0.2.3/.yamlfmt000066400000000000000000000002441521764210300150050ustar00rootroot00000000000000formatter: indent: 2 retain_line_breaks_single: true max_line_length: 100 # avoid replacing newline with #magic___^_^___line scan_folded_as_literal: true xgrammar-0.2.3/3rdparty/000077500000000000000000000000001521764210300151035ustar00rootroot00000000000000xgrammar-0.2.3/3rdparty/cpptrace/000077500000000000000000000000001521764210300167045ustar00rootroot00000000000000xgrammar-0.2.3/3rdparty/dlpack/000077500000000000000000000000001521764210300163415ustar00rootroot00000000000000xgrammar-0.2.3/3rdparty/googletest/000077500000000000000000000000001521764210300172575ustar00rootroot00000000000000xgrammar-0.2.3/3rdparty/picojson/000077500000000000000000000000001521764210300167275ustar00rootroot00000000000000xgrammar-0.2.3/3rdparty/picojson/picojson.h000066400000000000000000001046231521764210300207320ustar00rootroot00000000000000/* * Copyright 2009-2010 Cybozu Labs, Inc. * Copyright 2011-2014 Kazuho Oku * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #pragma once #ifndef PICOJSON_USE_INT64 #define PICOJSON_USE_INT64 #ifndef __STDC_FORMAT_MACROS #define __STDC_FORMAT_MACROS 1 #endif #endif // If PICOJSON_USE_ORDERED_OBJECT is set, picojson uses object_with_ordered_keys, which maintains // the insertion order of keys, i.e. the order of keys in the json string. // This macro is set by default. #ifndef PICOJSON_USE_ORDERED_OBJECT #define PICOJSON_USE_ORDERED_OBJECT 1 #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include // for isnan/isinf #if __cplusplus >= 201103L #include #else extern "C" { #ifdef _MSC_VER #include #elif defined(__INTEL_COMPILER) #include #else #include #endif } #endif #ifndef PICOJSON_USE_RVALUE_REFERENCE #if (defined(__cpp_rvalue_references) && __cpp_rvalue_references >= 200610) || \ (defined(_MSC_VER) && _MSC_VER >= 1600) #define PICOJSON_USE_RVALUE_REFERENCE 1 #else #define PICOJSON_USE_RVALUE_REFERENCE 0 #endif #endif // PICOJSON_USE_RVALUE_REFERENCE #ifndef PICOJSON_NOEXCEPT #if PICOJSON_USE_RVALUE_REFERENCE #define PICOJSON_NOEXCEPT noexcept #else #define PICOJSON_NOEXCEPT throw() #endif #endif // experimental support for int64_t (see README.mkdn for detail) #ifdef PICOJSON_USE_INT64 #include #include #endif // to disable the use of localeconv(3), set PICOJSON_USE_LOCALE to 0 #ifndef PICOJSON_USE_LOCALE #define PICOJSON_USE_LOCALE 1 #endif #if PICOJSON_USE_LOCALE extern "C" { #include } #endif #ifndef PICOJSON_ASSERT #ifndef PICOJSON_DISABLE_EXCEPTION #define PICOJSON_ASSERT(e) \ do { \ if (!(e)) throw std::runtime_error(#e); \ } while (0) #else #define PICOJSON_ASSERT(e) \ do { \ if (!(e)) std::abort(); \ } while (0) #endif // PICOJSON_DISABLE_EXCEPTION #endif #ifdef _MSC_VER #define SNPRINTF _snprintf_s #pragma warning(push) #pragma warning(disable : 4244) // conversion from int to char #pragma warning(disable : 4127) // conditional expression is constant #pragma warning(disable : 4702) // unreachable code #else #define SNPRINTF snprintf #endif namespace picojson { enum { null_type, boolean_type, number_type, string_type, array_type, object_type #ifdef PICOJSON_USE_INT64 , int64_type #endif }; enum { INDENT_WIDTH = 2 }; struct null {}; class object_with_ordered_keys; class value { public: typedef std::vector array; #ifdef PICOJSON_USE_ORDERED_OBJECT typedef object_with_ordered_keys object; #else typedef std::unordered_map object; #endif union _storage { bool boolean_; double number_; #ifdef PICOJSON_USE_INT64 int64_t int64_; #endif std::string* string_; array* array_; object* object_; }; protected: int type_; _storage u_; public: value(); value(int type, bool); explicit value(bool b); #ifdef PICOJSON_USE_INT64 explicit value(int64_t i); #endif explicit value(double n); explicit value(const std::string& s); explicit value(const array& a); explicit value(const object& o); #if PICOJSON_USE_RVALUE_REFERENCE explicit value(std::string&& s); explicit value(array&& a); explicit value(object&& o); #endif explicit value(const char* s); value(const char* s, size_t len); ~value(); value(const value& x); value& operator=(const value& x); #if PICOJSON_USE_RVALUE_REFERENCE value(value&& x) PICOJSON_NOEXCEPT; value& operator=(value&& x) PICOJSON_NOEXCEPT; #endif void swap(value& x) PICOJSON_NOEXCEPT; template bool is() const; template const T& get() const; template T& get(); template void set(const T&); #if PICOJSON_USE_RVALUE_REFERENCE template void set(T&&); #endif bool evaluate_as_boolean() const; const value& get(const size_t idx) const; const value& get(const std::string& key) const; value& get(const size_t idx); value& get(const std::string& key); bool contains(const size_t idx) const; bool contains(const std::string& key) const; std::string to_str() const; template void serialize(Iter os, bool prettify = false) const; std::string serialize(bool prettify = false) const; private: template // NOLINTNEXTLINE(runtime/explicit) value(const T*); // intentionally defined to block implicit conversion of // pointer to bool template static void _indent(Iter os, int indent); template void _serialize(Iter os, int indent) const; std::string _serialize(int indent) const; void clear(); }; // The ordered version of hashmap. It has the same interface as std::unordered_map, but provides // ordered_keys() to return the keys in the order they were inserted. class object_with_ordered_keys : private std::unordered_map { public: using typename std::unordered_map::value_type; using typename std::unordered_map::iterator; using typename std::unordered_map::const_iterator; object_with_ordered_keys() = default; object_with_ordered_keys(const object_with_ordered_keys&) = default; object_with_ordered_keys(object_with_ordered_keys&&) = default; object_with_ordered_keys(std::initializer_list init) : std::unordered_map(init) { for (const auto& pair : init) { ordered_keys_.push_back(pair.first); } } object_with_ordered_keys& operator=(const object_with_ordered_keys&) = default; object_with_ordered_keys& operator=(object_with_ordered_keys&&) = default; using std::unordered_map::begin; using std::unordered_map::end; using std::unordered_map::cbegin; using std::unordered_map::cend; using std::unordered_map::empty; using std::unordered_map::size; using std::unordered_map::at; using std::unordered_map::count; using std::unordered_map::find; using std::unordered_map::reserve; value& operator[](const std::string& key) { if (count(key) == 0) { ordered_keys_.push_back(key); } return std::unordered_map::operator[](key); } const value& operator[](const std::string& key) const { return std::unordered_map::at(key); } void clear() { std::unordered_map::clear(); ordered_keys_.clear(); } std::pair insert(const value_type& kv) { if (!count(kv.first)) { ordered_keys_.push_back(kv.first); } return std::unordered_map::insert(kv); } template std::pair emplace(Args&&... args) { return insert(value_type(std::forward(args)...)); } iterator erase(const_iterator it) { ordered_keys_.erase(std::find(ordered_keys_.begin(), ordered_keys_.end(), it->first)); return std::unordered_map::erase(it); } iterator erase(iterator it) { ordered_keys_.erase(std::find(ordered_keys_.begin(), ordered_keys_.end(), it->first)); return std::unordered_map::erase(it); } size_t erase(const std::string& key) { if (std::unordered_map::erase(key)) { ordered_keys_.erase(std::find(ordered_keys_.begin(), ordered_keys_.end(), key)); return 1; } else { return 0; } } const std::vector& ordered_keys() const { return ordered_keys_; } friend bool operator==(const object_with_ordered_keys& lhs, const object_with_ordered_keys& rhs); private: std::vector ordered_keys_; }; inline bool operator==(const object_with_ordered_keys& lhs, const object_with_ordered_keys& rhs) { return static_cast&>(lhs) == static_cast&>(rhs); } typedef value::array array; typedef value::object object; inline value::value() : type_(null_type), u_() {} inline value::value(int type, bool) : type_(type), u_() { switch (type) { #define INIT(p, v) \ case p##type: \ u_.p = v; \ break INIT(boolean_, false); INIT(number_, 0.0); #ifdef PICOJSON_USE_INT64 INIT(int64_, 0); #endif INIT(string_, new std::string()); INIT(array_, new array()); INIT(object_, new object()); #undef INIT default: break; } } inline value::value(bool b) : type_(boolean_type), u_() { u_.boolean_ = b; } #ifdef PICOJSON_USE_INT64 inline value::value(int64_t i) : type_(int64_type), u_() { u_.int64_ = i; } #endif inline value::value(double n) : type_(number_type), u_() { if ( #ifdef _MSC_VER !_finite(n) #elif __cplusplus >= 201103L std::isnan(n) || std::isinf(n) #else isnan(n) || isinf(n) #endif ) { #ifndef PICOJSON_DISABLE_EXCEPTION throw std::overflow_error(""); #else std::abort(); #endif } u_.number_ = n; } inline value::value(const std::string& s) : type_(string_type), u_() { u_.string_ = new std::string(s); } inline value::value(const array& a) : type_(array_type), u_() { u_.array_ = new array(a); } inline value::value(const object& o) : type_(object_type), u_() { u_.object_ = new object(o); } #if PICOJSON_USE_RVALUE_REFERENCE inline value::value(std::string&& s) : type_(string_type), u_() { u_.string_ = new std::string(std::move(s)); } inline value::value(array&& a) : type_(array_type), u_() { u_.array_ = new array(std::move(a)); } inline value::value(object&& o) : type_(object_type), u_() { u_.object_ = new object(std::move(o)); } #endif inline value::value(const char* s) : type_(string_type), u_() { u_.string_ = new std::string(s); } inline value::value(const char* s, size_t len) : type_(string_type), u_() { u_.string_ = new std::string(s, len); } inline void value::clear() { switch (type_) { #define DEINIT(p) \ case p##type: \ delete u_.p; \ break DEINIT(string_); DEINIT(array_); DEINIT(object_); #undef DEINIT default: break; } } inline value::~value() { clear(); } inline value::value(const value& x) : type_(x.type_), u_() { switch (type_) { #define INIT(p, v) \ case p##type: \ u_.p = v; \ break INIT(string_, new std::string(*x.u_.string_)); INIT(array_, new array(*x.u_.array_)); INIT(object_, new object(*x.u_.object_)); #undef INIT default: u_ = x.u_; break; } } inline value& value::operator=(const value& x) { if (this != &x) { value t(x); swap(t); } return *this; } #if PICOJSON_USE_RVALUE_REFERENCE inline value::value(value&& x) PICOJSON_NOEXCEPT : type_(null_type), u_() { swap(x); } inline value& value::operator=(value&& x) PICOJSON_NOEXCEPT { swap(x); return *this; } #endif inline void value::swap(value& x) PICOJSON_NOEXCEPT { std::swap(type_, x.type_); std::swap(u_, x.u_); } #define IS(ctype, jtype) \ template <> \ inline bool value::is() const { \ return type_ == jtype##_type; \ } IS(null, null) IS(bool, boolean) #ifdef PICOJSON_USE_INT64 IS(int64_t, int64) #endif IS(std::string, string) IS(array, array) IS(object, object) #undef IS template <> inline bool value::is() const { return type_ == number_type #ifdef PICOJSON_USE_INT64 || type_ == int64_type #endif // NOLINTNEXTLINE(whitespace/semicolon) ; } #define GET(ctype, var) \ template <> \ inline const ctype& value::get() const { \ PICOJSON_ASSERT("type mismatch! call is() before get()" && is()); \ return var; \ } \ template <> \ inline ctype& value::get() { \ PICOJSON_ASSERT("type mismatch! call is() before get()" && is()); \ return var; \ } GET(bool, u_.boolean_) GET(std::string, *u_.string_) GET(array, *u_.array_) GET(object, *u_.object_) #ifdef PICOJSON_USE_INT64 GET(double, (type_ == int64_type && (const_cast(this)->type_ = number_type, (const_cast(this)->u_.number_ = u_.int64_)), u_.number_)) GET(int64_t, u_.int64_) #else GET(double, u_.number_) #endif #undef GET #define SET(ctype, jtype, setter) \ template <> \ inline void value::set(const ctype& _val) { \ clear(); \ type_ = jtype##_type; \ setter \ } SET(bool, boolean, u_.boolean_ = _val;) SET(std::string, string, u_.string_ = new std::string(_val);) SET(array, array, u_.array_ = new array(_val);) SET(object, object, u_.object_ = new object(_val);) SET(double, number, u_.number_ = _val;) #ifdef PICOJSON_USE_INT64 SET(int64_t, int64, u_.int64_ = _val;) #endif #undef SET #if PICOJSON_USE_RVALUE_REFERENCE #define MOVESET(ctype, jtype, setter) \ template <> \ inline void value::set(ctype && _val) { \ clear(); \ type_ = jtype##_type; \ setter \ } MOVESET(std::string, string, u_.string_ = new std::string(std::move(_val));) MOVESET(array, array, u_.array_ = new array(std::move(_val));) MOVESET(object, object, u_.object_ = new object(std::move(_val));) #undef MOVESET #endif inline bool value::evaluate_as_boolean() const { switch (type_) { case null_type: return false; case boolean_type: return u_.boolean_; case number_type: return u_.number_ != 0; #ifdef PICOJSON_USE_INT64 case int64_type: return u_.int64_ != 0; #endif case string_type: return !u_.string_->empty(); default: return true; } } inline const value& value::get(const size_t idx) const { static value s_null; PICOJSON_ASSERT(is()); return idx < u_.array_->size() ? (*u_.array_)[idx] : s_null; } inline value& value::get(const size_t idx) { static value s_null; PICOJSON_ASSERT(is()); return idx < u_.array_->size() ? (*u_.array_)[idx] : s_null; } inline const value& value::get(const std::string& key) const { static value s_null; PICOJSON_ASSERT(is()); object::const_iterator i = u_.object_->find(key); return i != u_.object_->end() ? i->second : s_null; } inline value& value::get(const std::string& key) { static value s_null; PICOJSON_ASSERT(is()); object::iterator i = u_.object_->find(key); return i != u_.object_->end() ? i->second : s_null; } inline bool value::contains(const size_t idx) const { PICOJSON_ASSERT(is()); return idx < u_.array_->size(); } inline bool value::contains(const std::string& key) const { PICOJSON_ASSERT(is()); object::const_iterator i = u_.object_->find(key); return i != u_.object_->end(); } inline std::string value::to_str() const { switch (type_) { case null_type: return "null"; case boolean_type: return u_.boolean_ ? "true" : "false"; #ifdef PICOJSON_USE_INT64 case int64_type: { char buf[sizeof("-9223372036854775808")]; SNPRINTF(buf, sizeof(buf), "%" PRId64, u_.int64_); return buf; } #endif case number_type: { char buf[256]; double tmp; SNPRINTF( buf, sizeof(buf), fabs(u_.number_) < (1ULL << 53) && modf(u_.number_, &tmp) == 0 ? "%.f" : "%.17g", u_.number_ ); #if PICOJSON_USE_LOCALE char* decimal_point = localeconv()->decimal_point; if (strcmp(decimal_point, ".") != 0) { size_t decimal_point_len = strlen(decimal_point); for (char* p = buf; *p != '\0'; ++p) { if (strncmp(p, decimal_point, decimal_point_len) == 0) { return std::string(buf, p) + "." + (p + decimal_point_len); } } } #endif return buf; } case string_type: return *u_.string_; case array_type: return "array"; case object_type: return "object"; default: PICOJSON_ASSERT(0); #ifdef _MSC_VER __assume(0); #endif } return std::string(); } template void copy(const std::string& s, Iter oi) { std::copy(s.begin(), s.end(), oi); } template struct serialize_str_char { Iter oi; void operator()(char c) { switch (c) { #define MAP(val, sym) \ case val: \ copy(sym, oi); \ break MAP('"', "\\\""); MAP('\\', "\\\\"); MAP('\b', "\\b"); MAP('\f', "\\f"); MAP('\n', "\\n"); MAP('\r', "\\r"); MAP('\t', "\\t"); #undef MAP default: if (static_cast(c) < 0x20 || c == 0x7f) { char buf[7]; SNPRINTF(buf, sizeof(buf), "\\u%04x", c & 0xff); copy(buf, buf + 6, oi); } else { *oi++ = c; } break; } } }; template void serialize_str(const std::string& s, Iter oi) { *oi++ = '"'; serialize_str_char process_char = {oi}; std::for_each(s.begin(), s.end(), process_char); *oi++ = '"'; } template void value::serialize(Iter oi, bool prettify) const { return _serialize(oi, prettify ? 0 : -1); } inline std::string value::serialize(bool prettify) const { return _serialize(prettify ? 0 : -1); } template void value::_indent(Iter oi, int indent) { *oi++ = '\n'; for (int i = 0; i < indent * INDENT_WIDTH; ++i) { *oi++ = ' '; } } template void value::_serialize(Iter oi, int indent) const { switch (type_) { case string_type: serialize_str(*u_.string_, oi); break; case array_type: { *oi++ = '['; if (indent != -1) { ++indent; } for (array::const_iterator i = u_.array_->begin(); i != u_.array_->end(); ++i) { if (i != u_.array_->begin()) { *oi++ = ','; } if (indent != -1) { _indent(oi, indent); } i->_serialize(oi, indent); } if (indent != -1) { --indent; if (!u_.array_->empty()) { _indent(oi, indent); } } *oi++ = ']'; break; } case object_type: { *oi++ = '{'; if (indent != -1) { ++indent; } #if PICOJSON_USE_ORDERED_OBJECT for (auto i = u_.object_->ordered_keys().begin(); i != u_.object_->ordered_keys().end(); ++i) { if (i != u_.object_->ordered_keys().begin()) { *oi++ = ','; } if (indent != -1) { _indent(oi, indent); } serialize_str(*i, oi); *oi++ = ':'; if (indent != -1) { *oi++ = ' '; } u_.object_->at(*i)._serialize(oi, indent); } #else for (object::const_iterator i = u_.object_->begin(); i != u_.object_->end(); ++i) { if (i != u_.object_->begin()) { *oi++ = ','; } if (indent != -1) { _indent(oi, indent); } serialize_str(i->first, oi); *oi++ = ':'; if (indent != -1) { *oi++ = ' '; } i->second._serialize(oi, indent); } #endif if (indent != -1) { --indent; if (!u_.object_->empty()) { _indent(oi, indent); } } *oi++ = '}'; break; } default: copy(to_str(), oi); break; } if (indent == 0) { *oi++ = '\n'; } } inline std::string value::_serialize(int indent) const { std::string s; _serialize(std::back_inserter(s), indent); return s; } template class input { protected: Iter cur_, end_; bool consumed_; int line_; public: input(const Iter& first, const Iter& last) : cur_(first), end_(last), consumed_(false), line_(1) {} int getc() { if (consumed_) { if (*cur_ == '\n') { ++line_; } ++cur_; } if (cur_ == end_) { consumed_ = false; return -1; } consumed_ = true; return *cur_ & 0xff; } void ungetc() { consumed_ = false; } Iter cur() const { if (consumed_) { input* self = const_cast*>(this); self->consumed_ = false; ++self->cur_; } return cur_; } int line() const { return line_; } void skip_ws() { while (1) { int ch = getc(); if (!(ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r')) { ungetc(); break; } } } bool expect(const int expected) { skip_ws(); if (getc() != expected) { ungetc(); return false; } return true; } bool match(const std::string& pattern) { for (std::string::const_iterator pi(pattern.begin()); pi != pattern.end(); ++pi) { if (getc() != *pi) { ungetc(); return false; } } return true; } }; template // NOLINTNEXTLINE(runtime/references) inline int _parse_quadhex(input& in) { int uni_ch = 0, hex; for (int i = 0; i < 4; i++) { if ((hex = in.getc()) == -1) { return -1; } if ('0' <= hex && hex <= '9') { hex -= '0'; } else if ('A' <= hex && hex <= 'F') { hex -= 'A' - 0xa; } else if ('a' <= hex && hex <= 'f') { hex -= 'a' - 0xa; } else { in.ungetc(); return -1; } uni_ch = uni_ch * 16 + hex; } return uni_ch; } template // NOLINTNEXTLINE(runtime/references) inline bool _parse_codepoint(String& out, input& in) { int uni_ch; if ((uni_ch = _parse_quadhex(in)) == -1) { return false; } if (0xd800 <= uni_ch && uni_ch <= 0xdfff) { if (0xdc00 <= uni_ch) { // a second 16-bit of a surrogate pair appeared return false; } // first 16-bit of surrogate pair, get the next one if (in.getc() != '\\' || in.getc() != 'u') { in.ungetc(); return false; } int second = _parse_quadhex(in); if (!(0xdc00 <= second && second <= 0xdfff)) { return false; } uni_ch = ((uni_ch - 0xd800) << 10) | ((second - 0xdc00) & 0x3ff); uni_ch += 0x10000; } if (uni_ch < 0x80) { out.push_back(static_cast(uni_ch)); } else { if (uni_ch < 0x800) { out.push_back(static_cast(0xc0 | (uni_ch >> 6))); } else { if (uni_ch < 0x10000) { out.push_back(static_cast(0xe0 | (uni_ch >> 12))); } else { out.push_back(static_cast(0xf0 | (uni_ch >> 18))); out.push_back(static_cast(0x80 | ((uni_ch >> 12) & 0x3f))); } out.push_back(static_cast(0x80 | ((uni_ch >> 6) & 0x3f))); } out.push_back(static_cast(0x80 | (uni_ch & 0x3f))); } return true; } template // NOLINTNEXTLINE(runtime/references) inline bool _parse_string(String& out, input& in) { while (1) { int ch = in.getc(); if (ch < ' ') { in.ungetc(); return false; } else if (ch == '"') { return true; } else if (ch == '\\') { if ((ch = in.getc()) == -1) { return false; } switch (ch) { #define MAP(sym, val) \ case sym: \ out.push_back(val); \ break MAP('"', '\"'); MAP('\\', '\\'); MAP('/', '/'); MAP('b', '\b'); MAP('f', '\f'); MAP('n', '\n'); MAP('r', '\r'); MAP('t', '\t'); #undef MAP case 'u': if (!_parse_codepoint(out, in)) { return false; } break; default: return false; } } else { out.push_back(static_cast(ch)); } } return false; } template // NOLINTNEXTLINE(runtime/references) inline bool _parse_array(Context& ctx, input& in) { if (!ctx.parse_array_start()) { return false; } size_t idx = 0; if (in.expect(']')) { return ctx.parse_array_stop(idx); } do { if (!ctx.parse_array_item(in, idx)) { return false; } idx++; } while (in.expect(',')); return in.expect(']') && ctx.parse_array_stop(idx); } template // NOLINTNEXTLINE(runtime/references) inline bool _parse_object(Context& ctx, input& in) { if (!ctx.parse_object_start()) { return false; } if (in.expect('}')) { return true; } do { std::string key; if (!in.expect('"') || !_parse_string(key, in) || !in.expect(':')) { return false; } if (!ctx.parse_object_item(in, key)) { return false; } } while (in.expect(',')); return in.expect('}'); } template // NOLINTNEXTLINE(runtime/references) inline std::string _parse_number(input& in) { std::string num_str; while (1) { int ch = in.getc(); if (('0' <= ch && ch <= '9') || ch == '+' || ch == '-' || ch == 'e' || ch == 'E') { num_str.push_back(static_cast(ch)); } else if (ch == '.') { #if PICOJSON_USE_LOCALE num_str += localeconv()->decimal_point; #else num_str.push_back('.'); #endif } else { in.ungetc(); break; } } return num_str; } template // NOLINTNEXTLINE(runtime/references) inline bool _parse(Context& ctx, input& in) { in.skip_ws(); int ch = in.getc(); switch (ch) { #define IS(ch, text, op) \ case ch: \ if (in.match(text) && op) { \ return true; \ } else { \ return false; \ } IS('n', "ull", ctx.set_null()); IS('f', "alse", ctx.set_bool(false)); IS('t', "rue", ctx.set_bool(true)); #undef IS case '"': return ctx.parse_string(in); case '[': return _parse_array(ctx, in); case '{': return _parse_object(ctx, in); default: if (('0' <= ch && ch <= '9') || ch == '-') { double f; char* endp; in.ungetc(); std::string num_str(_parse_number(in)); if (num_str.empty()) { return false; } #ifdef PICOJSON_USE_INT64 { errno = 0; intmax_t ival = strtoimax(num_str.c_str(), &endp, 10); if (errno == 0 && std::numeric_limits::min() <= ival && ival <= std::numeric_limits::max() && endp == num_str.c_str() + num_str.size()) { ctx.set_int64(ival); return true; } } #endif f = strtod(num_str.c_str(), &endp); if (endp == num_str.c_str() + num_str.size()) { ctx.set_number(f); return true; } return false; } break; } in.ungetc(); return false; } class deny_parse_context { public: bool set_null() { return false; } bool set_bool(bool) { return false; } #ifdef PICOJSON_USE_INT64 bool set_int64(int64_t) { return false; } #endif bool set_number(double) { return false; } template bool parse_string(input&) { return false; } bool parse_array_start() { return false; } template bool parse_array_item(input&, size_t) { return false; } bool parse_array_stop(size_t) { return false; } bool parse_object_start() { return false; } template bool parse_object_item(input&, const std::string&) { return false; } }; class default_parse_context { protected: value* out_; public: // NOLINTNEXTLINE(runtime/explicit) default_parse_context(value* out) : out_(out) {} bool set_null() { *out_ = value(); return true; } bool set_bool(bool b) { *out_ = value(b); return true; } #ifdef PICOJSON_USE_INT64 bool set_int64(int64_t i) { *out_ = value(i); return true; } #endif bool set_number(double f) { *out_ = value(f); return true; } template // NOLINTNEXTLINE(runtime/references) bool parse_string(input& in) { *out_ = value(string_type, false); return _parse_string(out_->get(), in); } bool parse_array_start() { *out_ = value(array_type, false); return true; } template // NOLINTNEXTLINE(runtime/references) bool parse_array_item(input& in, size_t) { array& a = out_->get(); a.push_back(value()); default_parse_context ctx(&a.back()); return _parse(ctx, in); } bool parse_array_stop(size_t) { return true; } bool parse_object_start() { *out_ = value(object_type, false); return true; } template // NOLINTNEXTLINE(runtime/references) bool parse_object_item(input& in, const std::string& key) { object& o = out_->get(); default_parse_context ctx(&o[key]); return _parse(ctx, in); } private: default_parse_context(const default_parse_context&); default_parse_context& operator=(const default_parse_context&); }; class null_parse_context { public: struct dummy_str { void push_back(int) {} }; public: null_parse_context() {} bool set_null() { return true; } bool set_bool(bool) { return true; } #ifdef PICOJSON_USE_INT64 bool set_int64(int64_t) { return true; } #endif bool set_number(double) { return true; } template // NOLINTNEXTLINE(runtime/references) bool parse_string(input& in) { dummy_str s; return _parse_string(s, in); } bool parse_array_start() { return true; } template // NOLINTNEXTLINE(runtime/references) bool parse_array_item(input& in, size_t) { return _parse(*this, in); } bool parse_array_stop(size_t) { return true; } bool parse_object_start() { return true; } template // NOLINTNEXTLINE(runtime/references) bool parse_object_item(input& in, const std::string&) { return _parse(*this, in); } private: null_parse_context(const null_parse_context&); null_parse_context& operator=(const null_parse_context&); }; // obsolete, use the version below template // NOLINTNEXTLINE(runtime/references) inline std::string parse(value& out, Iter& pos, const Iter& last) { std::string err; pos = parse(out, pos, last, &err); return err; } template // NOLINTNEXTLINE(runtime/references) inline Iter _parse(Context& ctx, const Iter& first, const Iter& last, std::string* err) { input in(first, last); if (!_parse(ctx, in) && err != NULL) { char buf[64]; SNPRINTF(buf, sizeof(buf), "syntax error at line %d near: ", in.line()); *err = buf; while (1) { int ch = in.getc(); if (ch == -1 || ch == '\n') { break; } else if (ch >= ' ') { err->push_back(static_cast(ch)); } } } return in.cur(); } template // NOLINTNEXTLINE(runtime/references) inline Iter parse(value& out, const Iter& first, const Iter& last, std::string* err) { default_parse_context ctx(&out); return _parse(ctx, first, last, err); } // NOLINTNEXTLINE(runtime/references) inline std::string parse(value& out, const std::string& s) { std::string err; parse(out, s.begin(), s.end(), &err); return err; } // NOLINTNEXTLINE(runtime/references) inline std::string parse(value& out, std::istream& is) { std::string err; parse(out, std::istreambuf_iterator(is.rdbuf()), std::istreambuf_iterator(), &err); return err; } template struct last_error_t { static std::string s; }; template // NOLINTNEXTLINE(runtime/string) std::string last_error_t::s; inline void set_last_error(const std::string& s) { last_error_t::s = s; } inline const std::string& get_last_error() { return last_error_t::s; } inline bool operator==(const value& x, const value& y) { if (x.is()) return y.is(); #define PICOJSON_CMP(type) \ if (x.is()) return y.is() && x.get() == y.get() PICOJSON_CMP(bool); PICOJSON_CMP(double); PICOJSON_CMP(std::string); PICOJSON_CMP(array); PICOJSON_CMP(object); #undef PICOJSON_CMP PICOJSON_ASSERT(0); #ifdef _MSC_VER __assume(0); #endif return false; } inline bool operator!=(const value& x, const value& y) { return !(x == y); } } // namespace picojson #if !PICOJSON_USE_RVALUE_REFERENCE namespace std { template <> inline void swap(picojson::value& x, picojson::value& y) { x.swap(y); } } // namespace std #endif inline std::istream& operator>>(std::istream& is, picojson::value& x) { picojson::set_last_error(std::string()); const std::string err(picojson::parse(x, is)); if (!err.empty()) { picojson::set_last_error(err); is.setstate(std::ios::failbit); } return is; } inline std::ostream& operator<<(std::ostream& os, const picojson::value& x) { x.serialize(std::ostream_iterator(os)); return os; } #ifdef _MSC_VER #pragma warning(pop) #endif xgrammar-0.2.3/3rdparty/picojson/test_picojson.cpp000066400000000000000000000053401521764210300223200ustar00rootroot00000000000000/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ #include #include #include "picojson.h" using picojson::object_with_ordered_keys; void test_constructor() { object_with_ordered_keys obj; obj["foo"] = picojson::value(true); assert((obj.ordered_keys() == std::vector{"foo"})); object_with_ordered_keys obj1{{"foo", picojson::value(true)}, {"bar", picojson::value(false)}}; assert((obj1.ordered_keys() == std::vector{"foo", "bar"})); object_with_ordered_keys obj2(obj1); assert((obj2.ordered_keys() == std::vector{"foo", "bar"})); object_with_ordered_keys obj3(std::move(obj2)); assert((obj3.ordered_keys() == std::vector{"foo", "bar"})); obj = obj3; assert((obj.ordered_keys() == std::vector{"foo", "bar"})); } void test_modifier() { object_with_ordered_keys obj{{"foo", picojson::value(true)}, {"bar", picojson::value(false)}}; obj.insert({"abc", picojson::value(false)}); assert((obj.ordered_keys() == std::vector{"foo", "bar", "abc"})); obj.emplace("def", picojson::value(true)); assert((obj.ordered_keys() == std::vector{"foo", "bar", "abc", "def"})); obj.insert({"abc", picojson::value(true)}); assert((obj.ordered_keys() == std::vector{"foo", "bar", "abc", "def"})); auto it = obj.find("abc"); it = obj.erase(it); assert((obj.ordered_keys() == std::vector{"foo", "bar", "def"})); obj.erase("foo"); assert((obj.ordered_keys() == std::vector{"bar", "def"})); obj.clear(); assert((obj.ordered_keys() == std::vector{})); } void test_serializer() { picojson::object obj; obj["bar"] = picojson::value(static_cast(10)); obj["baz"] = picojson::value(10.5); obj["foo"] = picojson::value(true); picojson::value v(obj); assert((v.serialize(false) == "{\"bar\":10,\"baz\":10.5,\"foo\":true}")); } int main() { test_constructor(); test_modifier(); test_serializer(); return 0; } xgrammar-0.2.3/CMakeLists.txt000066400000000000000000000120761521764210300161010ustar00rootroot00000000000000cmake_minimum_required(VERSION 3.18) project(xgrammar LANGUAGES CXX) if(EXISTS ${CMAKE_BINARY_DIR}/config.cmake) message(STATUS "Config file: ${CMAKE_BINARY_DIR}/config.cmake") include(${CMAKE_BINARY_DIR}/config.cmake) elseif(EXISTS ${PROJECT_SOURCE_DIR}/config.cmake) message(STATUS "Config file: ${PROJECT_SOURCE_DIR}/config.cmake") include(${PROJECT_SOURCE_DIR}/config.cmake) elseif(EXISTS ${PROJECT_SOURCE_DIR}/cmake/config.cmake) message(STATUS "Config file: ${PROJECT_SOURCE_DIR}/cmake/config.cmake") include(${PROJECT_SOURCE_DIR}/cmake/config.cmake) else() message(STATUS "No config.cmake found. Using the default config") endif() option(XGRAMMAR_BUILD_PYTHON_BINDINGS "Build Python bindings" ON) option(XGRAMMAR_BUILD_CXX_TESTS "Build C++ tests" OFF) option(XGRAMMAR_ENABLE_CPPTRACE "Enable C++ trace (Now only support Linux, and RelWithDebugInfo or Debug build)" OFF ) option(XGRAMMAR_ENABLE_COVERAGE "Enable code coverage with gcov" OFF) option(XGRAMMAR_ENABLE_INTERNAL_CHECK "Enable internal checks" OFF) set(XGRAMMAR_CUDA_ARCHITECTURES native CACHE STRING "CUDA architectures" ) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_POSITION_INDEPENDENT_CODE ON) if(NOT CMAKE_BUILD_TYPE) message(STATUS "No build type specified; defaulting to CMAKE_BUILD_TYPE=RelWithDebugInfo.") set(CMAKE_BUILD_TYPE "RelWithDebugInfo" CACHE STRING "The build type" FORCE ) endif() message(STATUS "Build type: ${CMAKE_BUILD_TYPE}") message(STATUS "Build Python bindings: ${XGRAMMAR_BUILD_PYTHON_BINDINGS}") message(STATUS "Build C++ tests: ${XGRAMMAR_BUILD_CXX_TESTS}") message(STATUS "CUDA architectures: ${XGRAMMAR_CUDA_ARCHITECTURES}") message(STATUS "Enable C++ trace: ${XGRAMMAR_ENABLE_CPPTRACE}") if(MSVC) set(CMAKE_CXX_FLAGS "/Wall ${CMAKE_CXX_FLAGS}") else() if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") set(CMAKE_CXX_FLAGS "-O3 ${CMAKE_CXX_FLAGS}") endif() if(CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") message(STATUS "RISC-V 64-bit target detected, disabling LTO to prevent linker errors.") set(CMAKE_CXX_FLAGS "-Wall -Wextra -Werror -Wno-pedantic -Wno-unused-parameter \ -Woverloaded-virtual ${CMAKE_CXX_FLAGS}" ) elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "ppc64le") message( STATUS "PowerPC 64-bit LE target detected, downgrading -Wfree-nonheap-object to warning." ) set(CMAKE_CXX_FLAGS "-Wall -Wextra -Werror -Wno-pedantic -Wno-unused-parameter \ -Woverloaded-virtual -Wno-error=free-nonheap-object -flto=auto ${CMAKE_CXX_FLAGS}" ) else() set(CMAKE_CXX_FLAGS "-Wall -Wextra -Werror -Wno-pedantic -Wno-unused-parameter \ -Woverloaded-virtual -Wno-error=free-nonheap-object -flto=auto ${CMAKE_CXX_FLAGS}" ) endif() endif() set(XGRAMMAR_INCLUDE_PATH ${PROJECT_SOURCE_DIR}/3rdparty/picojson ${PROJECT_SOURCE_DIR}/3rdparty/dlpack/include ) file(GLOB_RECURSE XGRAMMAR_SOURCES_PATH "${PROJECT_SOURCE_DIR}/cpp/*.cc") list(FILTER XGRAMMAR_SOURCES_PATH EXCLUDE REGEX "${PROJECT_SOURCE_DIR}/cpp/tvm_ffi/.*\\.cc") add_library(xgrammar STATIC ${XGRAMMAR_SOURCES_PATH}) target_include_directories(xgrammar PUBLIC include) target_include_directories(xgrammar SYSTEM PUBLIC ${XGRAMMAR_INCLUDE_PATH}) # link to cpptrace if(XGRAMMAR_ENABLE_CPPTRACE) add_subdirectory(${PROJECT_SOURCE_DIR}/3rdparty/cpptrace) target_link_libraries(xgrammar PUBLIC cpptrace::cpptrace) target_compile_definitions(xgrammar PUBLIC XGRAMMAR_ENABLE_CPPTRACE=1) else() target_compile_definitions(xgrammar PUBLIC XGRAMMAR_ENABLE_CPPTRACE=0) endif() install(TARGETS xgrammar) install( DIRECTORY ${CMAKE_SOURCE_DIR}/include/xgrammar DESTINATION include FILES_MATCHING PATTERN "*.h" ) # Install DLPack headers from the submodule install( DIRECTORY ${CMAKE_SOURCE_DIR}/3rdparty/dlpack/include/dlpack DESTINATION include FILES_MATCHING PATTERN "*.h" ) if(XGRAMMAR_BUILD_PYTHON_BINDINGS) add_subdirectory(${PROJECT_SOURCE_DIR}/cpp) endif() if(XGRAMMAR_BUILD_CXX_TESTS) add_subdirectory(${PROJECT_SOURCE_DIR}/3rdparty/googletest) file(GLOB_RECURSE XGRAMMAR_TEST_SOURCES_PATH "${PROJECT_SOURCE_DIR}/tests/cpp/*.cc") enable_testing() add_executable(xgrammar_test ${XGRAMMAR_TEST_SOURCES_PATH}) target_include_directories(xgrammar_test PUBLIC ${PROJECT_SOURCE_DIR}/cpp) target_link_libraries(xgrammar_test xgrammar gtest gmock gtest_main) include(GoogleTest) gtest_discover_tests(xgrammar_test) endif() if(XGRAMMAR_ENABLE_COVERAGE) target_link_libraries(xgrammar_bindings PRIVATE gcov) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --coverage") set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --coverage") if(XGRAMMAR_BUILD_PYTHON_BINDINGS) set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} --coverage") endif() endif() if(XGRAMMAR_ENABLE_INTERNAL_CHECK) target_compile_definitions(xgrammar PUBLIC XGRAMMAR_ENABLE_INTERNAL_CHECK=1) else() target_compile_definitions(xgrammar PUBLIC XGRAMMAR_ENABLE_INTERNAL_CHECK=0) endif() xgrammar-0.2.3/CODEOWNERS000066400000000000000000000010661521764210300147310ustar00rootroot00000000000000# source files /include @Ubospica @Seven-Streams @DarkSharpness /cpp/tvm_ffi @Ubospica @Seven-Streams /cpp/support @Ubospica @DarkSharpness /cpp/*.cc @Ubospica @Seven-Streams @DarkSharpness /cpp/*.h @Ubospica @Seven-Streams @DarkSharpness /python @Ubospica /web @CharlieFRuan # tests tests/cpp @Ubospica @Seven-Streams @DarkSharpness tests/python @Ubospica @Seven-Streams # Miscellaneous /.github @Ubospica @Seven-Streams /3rdparty @Ubospica /assets @Ubospica /cmake @Ubospica /docs @Ubospica @Seven-Streams /examples @Ubospica /scripts @Ubospica /site @Ubospica xgrammar-0.2.3/CONTRIBUTING.md000066400000000000000000000074321521764210300155720ustar00rootroot00000000000000# Contributing to XGrammar We welcome contributions of all kinds, including new features, bug fixes, documentation improvements, and more. To ensure a smooth process, here is a general guide to contributing. For significant changes, such as adding a major new feature or refactoring core code, it's often a good idea to open a GitHub issue first to discuss your proposal. This step is optional, but can be very helpful as it allows the maintainers and the community to provide feedback and helps ensure your work aligns with the project's goals. The general workflow for submitting a change is: 1. **Fork the repository** and create a new branch for your work. 2. Make your changes, including adding tests if applicable. Please refer to the [`README.md`](README.md) for project-specific setup and testing instructions. 3. Format the code. Please run the commands below first: ```bash pre-commit install pre-commit run -a ruff check ``` 4. Push your changes to your fork, and **open a pull request** to the main repository. Please provide a clear description of your changes and link to the relevant issue if one exists. 5. **Iterate on the pull request** by responding to feedback from reviewers until the change is ready to be merged. ## **Pull Request Naming Convention** To maintain consistency and clarity, please follow this naming convention for your pull requests: ``` : ``` **Available types:** * `feat`: A new feature or enhancement * `fix`: A bug fix * `perf`: Performance improvement * `refactor`: Code refactoring without changing functionality * `test`: Adding or updating tests * `docs`: Documentation changes * `style`: Code style changes (formatting, whitespace, etc.) * `build`: Changes to build system or dependencies * `ci`: Changes to CI/CD configuration * `chore`: Maintenance tasks and other changes **Examples:** * `feat: support XGrammar with FP16 logits` * `fix: correct token mask generation with Unicode characters` * `perf: optimize the generation of token masks` * `docs: update installation guide` * `test: add unit tests for structural tags` ## **Review** Once you've opened a pull request, the review process begins: * **Community Review:** We encourage everyone to participate in the review process. All feedback on pull requests is welcome and valued. * **Approval:** For a pull request to be merged, it must receive at least **one approval** from designated code owners for the files you've changed, or from a project lead. The [`CODEOWNERS`](./CODEOWNERS) file in the repository lists the members responsible for different parts of the codebase. We hope this collaborative approach will maintain high code quality and ensure knowledge is shared effectively among contributors. ## **Merge** After your pull request has been approved and all automated checks (CI) have passed, a **Community Committer** will merge it into the main branch. ### **Performance and Stability** Maintaining high performance and stability are key goals for XGrammar. If a performance regression or functional issue occurs after a merge, we encourage anyone to report it. The process is as follows: 1. **File an Issue:** Anyone can file a high-priority issue in the repository. It is helpful to tag the original pull request and notify the author. 2. **Collaboration:** A project committer or lead will collaborate with the community to address the issue. 3. **Resolution:** If a quick fix is not available, the change may be reverted to maintain project stability. ## **Community Committer Role** XGrammar is maintained by a group of **Community Committers**. These are core contributors who have earned the role by providing frequent and valuable contributions to the project. They are responsible for reviewing and merging pull requests, maintaining the project's standards, and guiding new contributors. xgrammar-0.2.3/LICENSE000066400000000000000000000261351521764210300143470ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. xgrammar-0.2.3/NOTICE000066400000000000000000000000661521764210300142410ustar00rootroot00000000000000XGrammar Copyright (c) 2024 by XGrammar Contributors xgrammar-0.2.3/Package.swift000066400000000000000000000036001521764210300157430ustar00rootroot00000000000000// swift-tools-version: 6.0 // // Package.swift — Swift Package Manager support for xgrammar. // // Adds the `XGrammar` library product so Swift projects can import xgrammar // C++ source directly via SwiftPM, without a separate CMake build step. // // Usage in a consumer's Package.swift: // .package(url: "https://github.com/mlc-ai/xgrammar", from: "0.2.1"), // .product(name: "XGrammar", package: "xgrammar") import PackageDescription let package = Package( name: "xgrammar", platforms: [.macOS("14.0"), .iOS("17.0")], products: [ .library(name: "XGrammar", targets: ["XGrammar"]), ], targets: [ .target( name: "XGrammar", path: ".", exclude: [ // Build configuration files "CMakeLists.txt", "cpp/CMakeLists.txt", // Non-source top-level directories "cmake", "docs", "examples", "python", "scripts", "site", "assets", // Test files "tests", // Web bindings "web", // Python / TVM bindings inside cpp/ "cpp/tvm_ffi", // 3rdparty non-source / header-only dependencies "3rdparty/cpptrace", "3rdparty/googletest", "3rdparty/dlpack", "3rdparty/picojson", ], publicHeadersPath: "include", cxxSettings: [ .headerSearchPath("cpp"), .headerSearchPath("3rdparty/dlpack/include"), .headerSearchPath("3rdparty/picojson"), .define("XGRAMMAR_ENABLE_LOG_DEBUG", to: "0"), .define("XGRAMMAR_ENABLE_CPPTRACE", to: "0"), ] ), ], cxxLanguageStandard: .cxx17 ) xgrammar-0.2.3/README.md000066400000000000000000000157751521764210300146310ustar00rootroot00000000000000
logo [![Documentation](https://img.shields.io/badge/docs-latest-green)](https://xgrammar.mlc.ai/docs/) [![License](https://img.shields.io/badge/license-apache_2-blue)](https://github.com/mlc-ai/xgrammar/blob/main/LICENSE) [![PyPI](https://img.shields.io/pypi/v/xgrammar)](https://pypi.org/project/xgrammar) [![PyPI Downloads](https://static.pepy.tech/badge/xgrammar)](https://pepy.tech/projects/xgrammar) [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/mlc-ai/xgrammar) **Efficient, Flexible and Portable Structured Generation** [Get Started](#get-started) | [Documentation](https://xgrammar.mlc.ai/docs/) | [Blogpost](https://blog.mlc.ai/2024/11/22/achieving-efficient-flexible-portable-structured-generation-with-xgrammar) | [Technical Report](https://arxiv.org/abs/2411.15100)
## News - [2026/5] XGrammar-2 has been released! Check out our [blog](https://blog.mlc.ai/2026/05/04/xgrammar-2-fast-customizable-structured-generation) for more information. - [2025/12] XGrammar has been officially integrated into [Mirai](https://github.com/trymirai/uzu) - [2025/09] XGrammar has been officially integrated into [OpenVINO GenAI](https://github.com/openvinotoolkit/openvino.genai) - [2025/02] XGrammar has been officially integrated into [Modular's MAX](https://docs.modular.com/max/serve/structured-output) - [2025/01] XGrammar has been officially integrated into [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM). - [2024/12] XGrammar has been officially integrated into [vLLM](https://github.com/vllm-project/vllm). - [2024/12] We presented research talks on XGrammar at CMU, UC Berkeley, MIT, THU, SJTU, Ant Group, LMSys, Qingke AI, Camel AI. The slides can be found [here](https://docs.google.com/presentation/d/1iS7tu2EV4IKRWDaR0F3YD7ubrNqtGYUStSskceneelc/edit?usp=sharing). - [2024/11] XGrammar has been officially integrated into [SGLang](https://github.com/sgl-project/sglang). - [2024/11] XGrammar has been officially integrated into [MLC-LLM](https://github.com/mlc-ai/mlc-llm). - [2024/11] We officially released XGrammar v0.1.0! ## Overview XGrammar is an open-source library for efficient, flexible, and portable structured generation. It leverages constrained decoding to ensure **100% structural correctness** of the output. It supports general context-free grammar to enable a broad range of structures, including **JSON**, **regex**, **custom context-free grammar**, etc. XGrammar uses careful optimizations to achieve extremely low overhead in structured generation. It has achieved **near-zero overhead** in JSON generation, making it one of the fastest structured generation engines available. XGrammar features **universal deployment**. It supports: * **Platforms**: Linux, macOS, Windows * **Hardware**: CPU, NVIDIA GPU, AMD GPU, Apple Silicon, TPU, etc. * **Languages**: Python, C++, JavaScript, and Swift APIs * **Models**: Qwen, Llama, DeepSeek, Phi, Gemma, etc. XGrammar is very easy to integrate with LLM inference engines. It is the default structured generation backend for most LLM inference engines, including [**vLLM**](https://github.com/vllm-project/vllm), [**SGLang**](https://github.com/sgl-project/sglang), [**TensorRT-LLM**](https://github.com/NVIDIA/TensorRT-LLM), and [**MLC-LLM**](https://github.com/mlc-ai/mlc-llm), as well as many other companies. You can also try out their structured generation modes! ## Get Started Install XGrammar: ```bash pip install xgrammar ``` For use with MPS on Apple Silicon, install with: ```bash pip install "xgrammar[metal]" ``` Import XGrammar: ```python import xgrammar as xgr ``` Please visit our [documentation](https://xgrammar.mlc.ai/docs/) to get started with XGrammar. - [Installation](https://xgrammar.mlc.ai/docs/start/installation) - [Quick start](https://xgrammar.mlc.ai/docs/start/quick_start) ## Third-Party Bindings - **Rust**: [xgrammar-rs](https://github.com/trymirai/xgrammar-rs) — Community Rust bindings for XGrammar. ## Collaborators XGrammar has been widely adopted in industry, open-source projects, and academia. Our collaborators include:
[](https://x.ai/)   [](https://www.deepseek.com/en/)   [](https://github.com/NVIDIA/TensorRT-LLM)   [](https://www.databricks.com/)   [](https://about.meta.com/)   [](https://about.google/)   [](https://www.perplexity.ai/)   [](https://www.modular.com/)   [](https://github.com/sgl-project/sglang)   [](https://github.com/vllm-project/vllm)   [](https://github.com/mlc-ai/mlc-llm)   [WebLLM](https://github.com/mlc-ai/web-llm)   [](https://github.com/trymirai/uzu)
## Citation If you find XGrammar useful in your research, please consider citing our papers: ```bibtex @article{dong2024xgrammar, title={Xgrammar: Flexible and efficient structured generation engine for large language models}, author={Dong, Yixin and Ruan, Charlie F and Cai, Yaxing and Lai, Ruihang and Xu, Ziyi and Zhao, Yilong and Chen, Tianqi}, journal={Proceedings of Machine Learning and Systems 7}, year={2024} } @inproceedings{10.1145/3786335.3813124, author = {Li, Linzhang and Dong, Yixin and Wang, Guanjie and Xu, Ziyi and Jiang, Alexander and Chen, Tianqi}, title = {XGrammar-2: Dynamic and Efficient Structured Generation Engine for Agentic LLMs}, year = {2026}, isbn = {9798400724152}, publisher = {Association for Computing Machinery}, address = {New York, NY, USA}, url = {https://doi.org/10.1145/3786335.3813124}, booktitle = {Proceedings of the ACM Conference on AI and Agentic Systems}, pages = {1009--1022}, numpages = {14} } ``` xgrammar-0.2.3/assets/000077500000000000000000000000001521764210300146355ustar00rootroot00000000000000xgrammar-0.2.3/assets/logo.png000066400000000000000000005336751521764210300163260ustar00rootroot00000000000000PNG  IHDR9dv pHYsѲtEXtSoftwarewww.inkscape.org< IDATxku]ߘ9W탏mO`*(4I4ҀAMPDQoi#"cZ"LP Ԧ9g_?kεe=Zk9Xk ??~g^E,ًӔ>LI.$=d Ϟi{&[v)>#鳒dqd鷕GdCxp [W̏sɦIIjID̚.obڕ`v=ڝgh_עݕR8`\|dvհQA)qS%9>kzw69p}K}\z;;js>yy`nII8'{K-  >7X(/#&Lq'߉Sgq>w=^b>Oܸx]/|S@E 5Oӿdo.56NqU8Ɂkg{˧8o|VW$=ػ?iI+oySIzZy@4wKN~"_s?/'"g%NI2ř)gu}B{a3>OXd+)'O&>zbyLqNb۰Oqޞ|s88;Y|M8_rK~?.ou/{]z۾,ӗW)R'EXv_i2YݤɭxLqɤ_T/xp!r`G^ie7$M,=I8yěѮ~Vþ<1oaO0O!LEѻ߽D̼ѷb~\ɾd: Qb2Ź?fs8y-rn{?7´} Ox5I/Y?s+l ¹p=f}W8k l9=8ËmMq^=8oٚvK~#75_s1Iț}Oy)}[SC3b߉S~V<)%Kozc_7?pOzo~xg?%鵒ƙEvkZ|J YȺ5x-dݘ9Y ;V'}\NGVaS?a_+ߍ|V'ESׂp{g'y'4 p;oynʷ6~FS+'LqP)Z OLq>p)[Kso3_mB xRgzK,31ߊ)[Q9y>kS=>^'y/־z%mظ.Bn~_}ӷ>|Fp$Sb/Ϲ"$ [SpcyDz0]V1Uʹ&瘒L6ĔʹVGJRRy=rdmII6{NuT;Me7Ȥ>d$)M~HYR*JVIrz4빓,z~ijk)yè%Y9՟Gw^7SKIitQ>wj{KmoS&Mm)4vMRJS;Ǖ?>9=  ͅj5y-ߚ&/vkwpGq7׿3/O)SO O\bU3)Yc<ڵV)J\~yR=7é^ 䩿\b⩝kMVcf?WC,\K 'gN"⩄'z4M%,&ԃɟK4XzzLɇkJc%^I?J<%MEg ")=}ZO=}7=?"p<_SM3OLV-_#1pW!ɾgl,')ke8o'Չ-t%0iS.D)s4_Yq-ela2aK56~֒&?sݿ vnlԧOA%t6Mi*G1rԟkJ-tnbLm [x}0tN5t4UWkBiB7w8t1t.5D38]W9Zߒt(\>9:y/֧IaG/KR)~ <TfBٔjlTAH7X f-V"aD5D婤8H9i\ur />56Y=V?4C:D!t:09G)ME_#iATuZl[_\/>v4{/oOCEO{; 3؋ɞ>gy/9y+pLPnS\O`)&˵=ͭSbe}bP]_S&;s5|hr=tMuJ拡sjӠ:#!|I)أ~,z-t{WT9c|QbQ] _5}c=>eSWc>3~p7votBekk1MVI!lKsZlB6I~iJe+T-V叭~,,^5)?bdfR)k5l2Nf,^dYeu=fYʾ~}\^KX.Dzd,O)R.?zL.ó29NY$FIJfRݦ3_S*6&<X>۽:\MR}M˓}{MuӁS4]3}ֵ?esJTC9ŹpWua41>ޚ9oMq^/ +j(qg%=kvu-AsK}zw|ǟ熽??kඛn6|ao6(rkgXzle,ۦ\me5?+Kv֏k*+YƵYu!Ep\R[ۘn$k\_Y;fmsY y\^3e.+eSe9+=\CS=k*/M\zFίmN.K+f)dzRvv=r_w,e)MzڿO=>8/??)ͳ\q fc3W;,魒n69D6}5IJS|OtnLzNY=4T'&˔d}s&G&:Oz^=Վt89)u)}ϧ"^[F0>Yz }MtN>m:Mn1>]9MW>ѹLr>ܯ_&:)eDp?I.$꽉ΓŴ2yDt1LwWne b<;w,0?+VD‡>O Iߞ19oǗong5r1VNebr6Y)ss|K{SO')z?ڇRB)G>=:U-~:Z<}O_wJ=~kR M)=y=u5~v'Y4y{:O-d&=RoN&}ث?pLۧp{{%=_ 㖭) cs-+e.8cehs81+jE\؝ZYL2+/=v}V^k[\a.kOdð$\䲾Tz#Kރ[x߳l9ϳn9Lt}OSޛܧ4:Otlju*sӠnGݞl&:{u=UB.~5;7A}㟹זo[IW mĿ4)ǎ<ƗMqVj3Ƽx':Nw{Ku_D^nӝDe09ISR*Yv>Y?.$jlu2:esxMV<%MӅ|_Nd֔D0!"&FmԦAϚDgIuT':>NP˱6):M} sӠ/,M&ANNS,ݦD0 SM?O?_f- W{/>~X 8kcX#^z )Â椖-[Y꣝͓2NeᲙ)ij?oۺ>}9LVӘceMk }jO)n:ŸLRDg_?ڛ5>OV6':gT&:dfI%$8r^I>]9Nt6>9Ltn煉ersy,=L^&:Dg3Y}S+צzsO<7F 8[eo}%}oMq޸m[S7n}p4ZP[\i}lCA[J5ҭ6MR=ti%k%k ½|;Y%Nf:&IYSpx}--mP\BgFy CcCg?> 'NR:Ϧ29q tnSC:ԋ9s |P8 ef}xQ,DW$|5[8G޷1)έD<<.ӝKB&&'_G*w0s[l-';gcȼ:[;fSk\OJ0K=n!Ǽ-ǽ =z#k!tkCBgg j졳ι{Qt d&Bg<6 /~`{)55 p݈giwJO)?yjqsIBC穆Cg%/MOBgk)%Եc:={ vr\йG=T mBR9fu {:ocݧ8,bKs/:ฎrN:>}r%t-Dѻav=Cr5ޘ6Ԏ_', +"r}74y[1y OqC3{\1K} s =`N56s6}s o u tﷇ sꞗBg3dmBzVkIlOS=th;LcsQyܞ.s }-m%tm Bg:Q3oVȒٵɾql\/"gYy+%{joi1إ8?CS5%tD&:{< %%K5NmYmBgy۞a~m?8ζ:c-jriϖ˹1tnѲG:йLc!Ra òR^ '>Kx-c_ɡgw3z[ gepSW+i#^v_C”fsNi.}j<ξMJ}r+,B:caOTz>:yv<{A6Bg^Bg Sqlss}d>!nH#Vm#dI)Qr>L|Ϊ׎Aqz!NCg!tޭιζ:[׃ox4\ {U>Db_;ً5[׮]&:yOq^)Ωes-tml:5RLmmg 1o Ԧ1mZ]F-ԩ69:gйlrWjqzcܯI::qsu<pCgU:{ӟkܦE:Orqr99λSSW|k5\ϲןp\3l~$}Ie?=B=}i^S=jzԧ '3/Nt'1t.Ó,LN}qkm)eγuT'&OtnsBgߪG4*Cg BgSRvEVe0k/{`l}Y%*C$mBA㓖1ZᑱBAζ:k!tй{ =x\ù%tNO7x4?MK#r)x uLqrOqg[Dg8mR-cluZߤNt^ {&;Cgb:'_Ey!t:IyZnw6fsK0mi!t-:P85J\N[|(tA9!xn=:i%t΋_ڃk,I0fm\"gٰ_u䙒hss|+C\k1&Nk>a)Iڴ1tHU-~(Z%Y乆}{Bgk|mIC<βOenәgsjӦB~ ӟc\pkՐu ,tޕa!t=tC<=._C$Z>r}Oq^^S}:[seS/}brdf~ywS[i1r/ ӾV=B&:Pf sI1BgkF:4yi:J8\>%/8D1tQy}fsމ#/ pYDčo4]zs\:+54flj}bsVܷBB8ٷ3{s\~b%:c<vߺdv}F%tXX@-㶡Yl-7[ õ1Vn=sOйN0y9{ },ns3mzv|˾_ݿU9΂MCx+8|}o}ӧ%t5:X8%OKkcj4YybW9TՖ[Rlajs9?!Xѳ':֦>רy0z,Snsι\cΩ$:h~OCgcs:<p3f1uLq> 1S'w8echjGCgeZP&:йBqRN}\5ky&Iy MeZrݗ|)\yCz_e`ݢCsxޮ Y.:r C<^l)Yck':-t!t箅x1t+:x%dM[SggSo/ rNqGIS/"b}ʥ+ąCۦ12j M5d.tDgk+ s59BgTCgյԧ=|TC<~BHjsC\/c܂:٧I6y =Ps :s{l5t-P^ =L/|@7^)s gdgұSAOq^v)SS^ܚI:DgBgomO}V/`>Nй8ѹ,څйWs {<}:t{mBj9:a}ٯ q kP3yS}M` RC< =}C!jm-t6YJL}㹟5zw{q\3,${7::S1tV,y CgK|pr{zBgCgs?ΫWIZ dй=1>硳z&>:kxn'K =(::8cr@WBg[ cpw1tI;OYs<_6 YgLl2<DT?y:~1t.׍DgK5bB&K6YY>滭:claʳOnܔgsnsB羟0Dj\cdY5HOzpy1tD\sʦ&::_<釾o偿?$gpjz{tx~.SgX>orY':T]j>Z6Y!t!t.?LKh3zj@ܢmӑg6ѹ+մ1t܏gχك:[ }KLˮs-S[[dqt}֬ѳOz@s=BD[졳BT܂oWE 8[o=^!b>)[^zs {NSY]:J6my&:Y,=~A+%5`-twjS& r[1y/<>}S6wckIdendJM9jqr-˦jաϋsi[5-*OQnpX2>c)os[,וG s=d^ s rv}snZZcY-t:Fέ}7K#r[1y=݈.=9ۚ|zF uZX\:TF;} s<Bg,~ms SMCd)k<N,tN1v&*z,Bg롳ʼnΡdn=VD3r =Ny=..oy:pչ]:gi!Jn\,Bl5\Dh:Չ-tná·BױfJyL_|r\?"g(m/7Ӧ8-eqyh؟+LtGBC:[\M|_ȏ)1T0Xxُs=N=tAe\ 4f!tClC~c\&9kc.wgۏp>|e:y{]gcIr åfIbyn ]lja tUc)/%2&:GB9=ZLFyX=W ,u1t G9{\-M[Ndg+v14 IDATΚy 56aYl:~g'}'/}.3L-ĉ[SN}=!lxg^^F s6Kͪl^.qD:2z g!R.?mdžqjιϯu/idIV^˾~ ׆كYcXfsHY'AY.+#Y<S ׾~WBpN Ozp-{,;K1tsJJ5s9Ϊ>m9 i+1&eJYNqM:B09 :^oµ)'wO\D~vTcd:ξϝ5UCgOtoc^x^Kot ey SGu!a³ivC쓟s{o{OYy)gY?B约jDB)ygf!N)Sѣqµ5^rJhm{χ^йD܂:k R}2sK@Czz^fV#< {u:tk,MOB@ { K0\k1'<'y4qt=ss&7H9gep\nks|r D3׮SOp)+68Oqp|>x:Qɦ:^)8Mt:h^JJf5s9X309ׯBzFlpmm1zd\J쁰BU˧!׈|[)Clu,YsCg 5q tӢ}j9Sn "gYdNqGW8p4Dg)%Bghz =`YYj&:4 :{l:%K5D,t=t-q}ju f=.s&F}Rs6SR m D`\E1Lq8+D3q8SWC#{)s|CSK|D:8>uy'#V̳&uuS>yAX-ږB mfa:Byp\sC-ys!t!t:[VΪrOoB?\gcY!N9⡳نDԹOwq8}e9Ό͢ڻnjy}es:D^ǖRD?ɤ\Mu:[^S%czl}{{,B^ZS{\em6}&29؟γm*s s m:װ96iι./=Me9">wkݹLq޺n<CZ/9dS m:skBa㡇YZamGI'0kBsZ~N\%)I}#6k\d;:'lVsCB~~C(9Έ1yK)'׷zpOtV\2yXny%Xd)Omb>9KCtڔ:[ 5D}:s G>g׮M^.s,Fuݶ1<>Y+s9كj\_uOs -Bpq\⼷sB5W[4{h>?Y=6aM+q Ω?s/S|D3aWz+D9y˄קOqcxk/֭\й%UlY%tnzCgx695")̖5JVB\>3}fW':k -D-hCs -mbsC]Cgks朳7gۆpOcA1إ8ϣSbaZQqkcKْr '%IYJY9йFĵ.r $|:{wr>zCDhSf_+D˾G8й=WX{+61Κ>C\眗B/SOBF 8GO\] d78_>n1)k/K)k'⼿ay:[9]Ou1k %jqr$z PC~]=k d2:!t8~s 9n6"g9jףǍx~j\f?>nM=澷dc>29wAy}?g>37fᖁ! $SBV˜ fECxXct.On>o9%t(tV+&tV)й]tuhslgcBgQouf;Xq'c}ex!];9EKZx}7z>E|~'^g%4c|йY~Ρs}1A&Cyci..܆ݢs(+ K lBg59Nmeaz.tNEЅΡ =4?]q&8"g Vg޽,8{ɟْzl4ĢnZMl%t'K7{sD#]s^^C<Շ^Bg)1%]ϞaS:9-;H :ef C9Gv\q^Kp'Z+ΫGˊ}g_A+-L0 C|5RI\t!s C,ɼ.:Մ. )ΏBg57AOoй[כwGΡs(tb*'lG kVcr\tLmʵk<ʺBeE9yR?3Dq\cysw5Vט=&V{Ңs )B[:[YtVsGs~iqB8ݼWCX%tV =xk 5V M,/aq:׭Jss\^t 9teZiA:_Jss%kaߒ7'xOkDXq^zey]WskM܅iY}n:{scK6d1?Q\CgZבSX%C2K5t*ss$??:5zzsws\;GD 8A㸸wr!&;ƊN}OqŹ;Yt%h]pn6(:΃Rl:h=1[x> Sdl1vkBz>й}^Vs lCgbɜBg߀t<6/Oכu::䲐 Ab]0>xpӈ'5VޅsotygbŹ}܄2|bYS7EA0YMܾ6t^ r DA*+!=^3CA9RެT:Ίr=/2[y^ ?!u:wAts]cਈ'f9jdy[q޽5L?9&)4Ζ"&t8ߪluAй6i}9t5cJh?8ǸSjJs|:t!sX$!^;(BCD7s}#G?["r仹܇{}W^c:df1&B>בL}O syYݛב.ՇR 9LB~9TpkCr_S^C0{:41^xΡsg KmΡ)tgD 8!QQWw>+{޷o5O=^\]>w=֞ YM`܆!>tWkCtٴ]CgoR'+Gքm,GWy ^֐MyQrsYxv\jw+jWgCgMyg8IDrBSBuy> >l8/-T~yR)t`CTzVs.+ssњjkCP_gӋ9/_&C]C0[ s[Ke뱓94s]xft9>v.j6=$>gu/g{8/Y8+k̞;3kZtٚn^>:.?iQn2:KeYh/C;CgsYgnC\?Ay977s:.8йяgp:C9Ks]Zq^zeyos^z6FGR:+_8ӄΥwvinѹ?wsV8_KsІ3?Uu9N+>Y^q.KD,йYt,:BYqN/&ǽ?u3/ M9E|Nܻ:==SV--MBzLk:%P{ 9n3)3L,81{SZok9ȣй2{\ЄtͲ\g: DNJZ{'W3sbz[5okR¹ul?,]K\:q6CTC&mbtn9rγӚr CY.?':sfC^v5Fsʜ1: i:?s ͯ69N!ͯ8o/ޣ _9\cw>ܥk\5O1X>s ]bHl9xαky^rnrdq&\t sוE\%BH zs 1<9.s KH/7o9ksΡ{ pBYnGsa+3vZq^2Oc}y}obytQ謲:hהB2r{ sQ5tNqtGR,s:7K)J[}nxj9}\B=Xq8D]lֻ-EOgqyZx8Oc-wrj1tx9塹Rpl9%vf:ΡrjkS`:7%t^5&֖i 1hCxNIAv c:{:/<Ǐ F 8{8OGƵ8o]cا|Ź=,Ρ S\CnѹW5UCgEvA9/<:ӷ܆ήtsh^GRsHs^MйYt. c7ʊ3"r<=.ˊıXq8~jŹU]tVa"tzr++9ơYtE,T":˛E:sJ\.bNz:sk,)Vrs\Byy:{^auHd~^]w"rA1ׅx+;ϲ&W8>fu.1nq )l9UEg+s|% $Bgus~ænYv.s]e^5:{,8x;8 3pPu FWgeyYqo..dm:[~Orp\B%tRr\|>WP=J]ctn>z R:56 b s輽ڜ"ty_&"g;s>>+k.]\q^{K+ۡ&tFs?kpKsseclA D\9F}E+ʃrF!]蜂@7s u SxC8s8p>w3A 8A{k+γQ"QA+{޷vo5O=^]q?w5Bs_G>L^Z:Eg-:cf_v9>.9tNArp8}x|nUB||[zn8܄5t\!SW=ӆR: 9n3ČQzhtd[YbgG ·,+Kv ok~+γϨ_tV<[56(tKC%[FY>ƞ5J,B{89tVe~9³TC'Cgמ|8UǶrk;ocdܵ0xʊw˛[ǡsy7޽5ۗz"tNs ʁs:+C9;h\SeC:t^~ˊ39Nxxs!#+ά8/|,:7+9N ]E 9Lu9rTކzs(ksP]C,:E,֞MX:_/YM>t,;~&:q<n 3)1c8ˊs:5rls\.Y^tn:_tHDŽ1B*55s$)ą tk\9}o]ߙk;p"B8cOeyأ8WUw7|sw Pf6~s^tnI:(ښY VǡsirƫބZ1tM,MAa2nB筟}f"r<v_&fyjĊڹ{WV%w_ٶ/[b&LvAuǡkhb89=r܅R~Bv8n*+pzVފNoŹOλ,:Bg[|޺9>Wc Ex'w*2kCڬk >CgW:͞ *3yVN3uy1 8c|8~veC=UC^1r:spk:7+˲~]\Vmzѹ|&t:}<%.sC|:ǯ plDr*+kcy-]8iYq>$>֊s{n  y{ە wΡ,?Ƿ&BgmޞkEބq^./D}lry1sa72E:/n3⼋gyŹ+0We9rˋk1tNsq:{E)Q8Vjtr<]Vki^c>{Yq8DϊZGļ̎.}fV^q^r+kޱɺAf+m輽fr-: йކ%tn>¼:u C;|Z,(}axWup'c:H/N8/+γcV'}+㧂[g91(5 }2t|XדڜB܇sF C:{:.jxl pJ'fq}y܁xs\=Ɗ5|jMf> s4跆0 Zr)k&txo\gsl9N_+ S!ΏsHsoiy_pžs.8^<~sV3LCgxPYtT"e6tG^]qr<}\Q#r|&Bgmkjbd9N+_q>$~W#8k}:csڈ8]j̬c<vY6zv}kϱv^uΡsh֟CZv̧~KXq8DŊ=8wtcYq~b+3ܞ5tY%tNs::{:ZsxLH=W[uٹC:˻95tnbhs|-8'Mû77^gEuFvhs~͟}| ʊZԻp.+cyG\q}XqnύsˢGæ:_N?Y.3{J<.Vs!z|Z,|1u|~=2k(Ad.τgz*#i+?uk\z-&)I+S0I ._yC&KpVgw ϲ1ׅx+7>,8ϸ+γn ϡr7%K_EfѲ -9fWsJUd5ͱ1|t ^WcY a!~ & AKy_p߰DZ8S9>pvMnFr'I"ږE?s^]Ga.\ϸ75/788Ϝˊ͟i(ΞBzljϫ5e3sr9̬ MzUHKӲK~yb.9YJs]z pwo|}u~;% 8ZƖ % &e)'c>f?[SkǷXŊ=8wtc3wb>絟̊N}o|TSQ=˔C5E2\CgLzHq^t.֟I[oωjN˂U>dC KW#x8?{r? 6he&}گ|y,:9dC Kl{ Ac pj| ϝ=Cwtp/p=~Wo`̛+Law^kyrE\{<Cs_Go y ,:ל?90ykѹ.9X9+6]rOY|6tNǩ $ɬsˆaΊ3) }-{#ُJk8/wJSx&?Wz"`ʁQdϊ>qLSvZڕn"T=gu/g{8/Y[q^OeŹ{+qX֓78tN!qtsMm\zo\Oar:M o^g_FӼ¹wO;p|>ů?Q?q%ݛ?V5ݘI>_y~<MwZq^8uE tۡTcgIqmb,Iq3+λx/c.YxIw;Xq ??lO>@+SYq98,8NJs7 9tJq Gn%)[~YVנ]!b G*J߫I88 ioz.I_?Vg;wqewox'_;fWSYq>9ySE{\zG]q^~A玮qW=Ɗbh8ǏƒyzyP)NE NkMlr;w翈 _ {_L^ 9n1n=g_q;=a+v`5<\gMgHvߥsBZ- R7\&=kױkcy\{0{Kϼ#{x#8ˊQV W 8IΙYi/ 47藃>sݍ\oJ|%]z/I\gӲWx͊suk+>xߥl Bg+?й+UfY=!DB8E:BK[-:yo^˭j3*i؎n  :t5OYUNz93oih,R4s>wHvL}g7k/k^nf rُ>un=;nONJǮE >}+vIf-80 KlO;yY5t,+9f{/+& joj[Kߌ: 6~p[?a:ܰs&# yك+ R},P1`~%`j蜏qzugz]g9J~o\/~',D7/K[_;] }!+{,oww->/+iP{zyy律HB c#fsz,:Y)hNYcA},-).K!䜧sCmM px v]՛ƆAg^0Xl^B< M9HDs cO|,x+֯3/x2pnc\_{-Rpirt:? p 7y`rb IDAT Yqvٹ5Wm` c༑L⒳o>˯L~+Wظ1>e_JRˠp .].c/$ KgK_ݯѻy{S6_{Ź56?d[U1d>swb{,;x8/Y[q>]݃yXo%gS=E1DxDڃS˝%xft|~>7.;Bv8?`z0U7k8Av?h6Kf Riyy]Q謭YqȡUS|i\(k~:!!q]Io?|Źk<|q2iS/_t:+.>7sqs4C|j$}f8G_/q!xJ볬8VW 67Ǻa2tcdOs1dvEЅrN DԇϬ8tTte5|T S},gW.t9GK<׻k3oﰫ{'=̊^f|S\-49-1̛E:7Zq9FˡsްYtV:^{:{Yҷ|8>q,qU Z?ptӅΒPrɂ٥:{a| A1dnWK::9t2/R9kuQM}>;:y^ͯ~_|qA8q8OZ[q^zey 8ǖйm,:9MlcHsKf!bzm&t _~' p79?"関˭sY қQA)tbsb]Cg ͕7_1_ˢ_(t>t~ڬq}'^ D7Xc.L8o] 8ϼW{+γS\X:HY:׳Y35M> Ugo1 |w}:(r~efWn+ɡs eR,!լ:M,y>tޘ {7o_N_q9繛߁s؊z<}}LM'2 [t]謴݇g?9tvY E߅ | w֯\sy1i.AnJ} P::e1Х!α_NPXLn+caР_Gik,־!|x5}/oMok(EggL.߫Ww=絟Ҋn1}g^39i^>?G-Ȗel.J2*I,yNr7 9`L%`r]pgjWMtLtsZdtM,O 1!,kLξQ\aB+Ki*<|e\~i ӢsYlp7+.:% Wx cEͅ˾}4&Jm}߻p00]q]\1^tvyby#ic:o$߸JҕRCgK5ttePP2ΏCBgcח% ݎ+\3?Ͻ}w=B'BM8<z}z~yoy律++򼯜ϊohi}O>vmK~Q ֜soKz{^.-R?/_CՃ݊o>T_㐘xwwz\V9͟;ʢ7ɗ?yϋΡ;gyf:g׼3 1F?lݱ7oכSØ*)vOqsI)CgOT.tVk,O39Ϗ%^C#6\>q>_x; '`SWW.{VX{6+·O3S=įY͢s{GwiC:ᒧ:啳 prV#ߥuw_}+ɞbAy/J L] )Iѳ6RmWk)r2keTǷ9(\H=t}z Y*vZﭾpg]yq&&޺7s6&>x}]qqk;5ncŹKe뢳d],տ&d/٭=y5|/c1rVopIoscYd%tܬ 9$ml9FW,<65oM: ͅ!:?ЄAvO_/-w #/W~2AXq.Ǟʊ̱G]q^{f/8{k8wˇCg(t߹9DйF͡#̟=_̊3/=y$(-ݸImfzlQ\Ǡܘ|yAQ :5/,rŪ<4ȵIϐ *׉_37W{GNJ=M+O+n7/>絵ü,9u<ݤq0}B\nq)-9F +O.]•͕u+% )\.\~c_c-:6]qWӽ738/wW-+λcy݇]c#8s,wihRg~_`S}ey}]Zq&xm /_}r+~VBn%p )HgHQR sW1|Ρ9=Ρ$T /=W)^ M % R S"8hu͗bl~`08Ak+΋OrPsG`YIZԻ˩8k3[~M8yZps͡7{`ΡnqAtb3397a2r97wxy[RÚXBgƤ֜\%t +{ZvY\k1tV:?sxz?@/:?8Q7v.+GYq>$~+73Y[Ce)nVҚEgoc܅3Fpk&#g潒|a&Pw|s[ aTBg /:+~nSl1zv ,Λ6t4e:!Ηmם7|Mo&=~Vv~@wN\q+fŹ+λonZ]4ocܛs^tΡs>Gl4[y9BcLGf߷|>+;>k؞|?qYaK Ai٥M%VV?AaT載}MK:? a:£/=כ=Lc|Ͼ| wLxͮ81{ZԻp\q^ V<3Y_>k\wgrWй,92.ikѹWBgog5獺ysbp Js*_ކ)|~(;7x:RZq^ R¹u{Pu/IYjOwsGwjYV9Cv;VND;]#P?)sl$Bp):o$|g6tKM7_2>.CBZt:Džg%t)tǞo{~y~$7{\F8EKCWpw;wR _q^r+kc+1bN;ff0HKOL&xCrwcά8.r~P|a5r+͇4&_+Φ"6tnk7{Zu&tvҤ+qKWV>W/v?6tG1t~y Zq~^ɯ;br7^/Ts<Oc}y-8|.+SƹmŹ;,:8[zB)t.YowͅYq#/hϸ~W<Bl`hRm 5gQ5 Oxfb\ R,cy~>$ WH7d $gIA}^7>d"J?WZxv[ڕ 7w^q^ڊsa>'8ޗ=sB >l+<5^\C즧nŹ?e2/_LbLSz6阍Egmכ7q9.:u竴| WqYRך/s\y.\{Zvv~^oΡJQWo;+Κ5?-+·ׇ8EgfYz`(+>uZqav89>]*1~+ߛDx紵8Źu-rA8wltV ZeʣAo[ @B3\QJM&0qs8mho :{AQK5o :GVй+\qr9l :{+J+p-ʗt>nEےUy/tP+xMsziq $Z;UVYka{]AP\ 0!g#Ĭ]]{sE!CaE03iqLwsNJ& :7сկP9K`T!g.[;C5 :Etzլ65I+.EaVйlxl WBr=ȼ7׃KFZAP2mڔ&0L AsB[cU㎰9\MR^P&ڶ[f˴t5jf9qj!Z1}Έ-Ν7ԨeICSDf !gYl^/ %vl-LҊ+6)}BRg7CzfeJ՞҄8w찞G]r3ɣX&as Dgsi:9Jkt{nQfo泭$UYLg\NE0瘵8| Z=f0qe]e=Cz_xyk~G֗涉3 {la$(1`sb('ym aM0Ljn ,^C.Y=lĎPsYTp%P`Y F9T kv6#A1BI[O/)d ?WXZn N[ 7{L@ Q~3oVYAfs%dyȃ+)I)q3S1s7&dnqiikfzZw27ts5o!@z@GZf& Ճ ͣpGK@R(U¢MMR(y\^SV$_ilW,˗pKR$r}%/%)\ ˒!GHE IDAT 1ϗYgĜ(^j#pl_-9;` I-k?wZǷ8S|Iz-]w+W]t6spn˽kB7i!vE&1c-5Fgt6I,ע}:iqu\Z{]6tn5:6A8ϳe {dgc[2pLܣMѹУM y({oФ`{ڂ5Q{s#Me͟5 Gh(<5HKwS>g$,;[B k7Js?B[c{n}lt9SorzKADwOZה+<#1yñkqn߶l237:[AgST5&\Zf8bբ&(rظZZVlz $CfӦْ[۞@A6mܽصc-W﬏]紵 -Ν_gEzճ'kqnxXkSZQ5(< 1A3Q(%YXO^ot :[ ۫=lȭVEe5AgM%E2Ԏ(_䌞:jqٖ礃✶硶8Ρ޶F:=njFmwj-Ϋ֣lqYѹ-rMX_۷v%֊Kk*Tb i)6[=ݾET]j8KSy-α*a--=Ąs'Ź}m9)u n2nym׼Yi'olvW~IaQRXo0trw5E},Iۮpţˮ,jK.-K6',3]ejs[W3N`?3Z5$8w#vܮpn\%jt..|-䄚0%igfmqnZ/-]pMkCMt#8m%oW'P<.zi`Um얽9iWi-IisZZȳ'-Ik9)I'%maZzl9 07:Kx9BCId ֈ7{PLA' $'{Ӳ8,ڽ6sW1za[Xۜ#embcpZZ{uB>87(ߛcÔIK}@m?tchVvLmA@xѹ@ZFZ]At=,˞}fZ+ޯ28'#[HZskZ%@\:d]Ak~ܢsѹ1oZ۷hktnijxm۴.?1iqn9C({P]紵hqN ✰Zv$?y؝9v 1g8Uv-ܴ !vN.mjew&SUfIݕ%mz:l,C2 mIC.鐼Rx%/{GuI0eH:3~Zksh1}hs=,k6[VS">&3v,㿅 78kM[ ^T+ |\L]-zjQtwָ$Ϟ(5,ole2ܾ,(,}%jwL)c(PF<ҐC7(cnR@rs=l%ΈV|Um~^3yLZ{Ź/(Zgt-9 !6H D7{'uV+Ͳyr=Ln4/7Bnw-Us׭yww0vNKjb^/-iF0èy]&SC7y負`偢g5AVY&i\;B3n8.9\:[c$8egLjqN{&͜cm!C9.#Hvټp Yree$Ug 3.Oy}w|mL_-ńύH%]APoeʟAFFع7εcfL) b[8XMLIk!~&lqZ| {Nmۢ6L/$V>[޴H{|ߑL)Q^oqt67cn-><$z|٥ AGAz{sۃʑZ92`-)KHJ~mH;^ir7munIVw\(؜L~BRWf+ _pӁkl?|!JKpPosyZ3}B7(ńnj;YG9փzY^:[g$3j^;%/Ic~MZsiq6Z~'I3x.{Lz:d}ki!ׯݤ{k@_f^X ./w-tj0&$=\O_lerW>j7;QebM$=cK[;@hy?mqugsP?a*M ڃ΍CY y=gApq8ӣ`\ 7:ω]ܴjsOjK缦N9-J~v| ߐc֘yA!OrYY&M Xs/v$]Jg7VXJ*SjُU [X7|ߔ$ok(8{ݢFf9(mRzs\j3g`t\#gCpGrv_ނ;gf| n9:kqN9nbsڮ8e&YR`W.0׫erf\_$X_f+ /v7=$r释3wsBg%VLK6 J^C){$I >BC7;E.n)KK% B5:$j}qiq0y~gf_kkqy\Z縃lwcolEG~C]E1Lg,zw_]/z G!1\p\y¾mwɞ+r=V_ >V-cgK\h#R~,;F-1F8'IkqNzMkqNZLc$Os\PfZMz^Cܸ|LRsLp7ѩpԢ@Ka)xEC`c[\eៗ5u]#ӂRѳa"ϙ]̕fGe9fl*z0x $i֛%l!җFzйjn:P\ǥꍽqZ ^svõ8e2aL2;s,:v?/E1O,z3ezesL$7=u=ֿG۪Vɗ-iٰnTjy볕?ٱmB3ڊUIZpm8g7΍Fl2w{WٚA Zlp 6LmZv\Zcypkc✶v-8K:8W;ԝ8arIGcM]SVk%m)z ǃ=a˛fJ\\X"ea-/av3]P4nZj>u :͠\ma=l By-5-6 65nFsZ3^P&ib /.u)Beo(zfhrsZ$]V}Gl§kax̯蹰a=\WTlyaNK*E(V3SKk✭Iҍܽ{tvϡGfй|{ř3w*{CL8>g(-έ,ԝpX2Zzٛ/zI~os`}yrf_o'\L(&ӂD7*cE(F3|ͷռ-vo;-k]rsmAgIڂQR\5-6sqs̶#mqgf_k}Į Osb~'v>c% .Q_(zje/@+%;Hefsf+ OIۊ 򗇡2[Yy҇9墇NqϔH(l#͠Knrww rͶnzQ89-]9M^v^;J{G _f;xr9&۫wW.810>P~9&#90ffwV~Z+#Ex??ZsەʞS:BfaWy89x5GCmAzs(JLׂ`L?]zs?B[Z~'B fnQ^v񀭜xsa St9&.zQR9&зK7!0=k%jI/z& ݒ&wN/쭖+z >kYŹHCc켾]Q9.Mhږ;wU[%{8ws1^1aUH_7}mgӟpa 9oE1LzqEρV.zI/,zIdz3 ~xP^;z|vzs,WXV7FR_w`0qnZ54W@Z7AgÞK')-`N+*qsPdw8-i'9ŹHnqNwelomstWkscc,z ff&ROLZ/EV+ﮅ-&Hõ'S-:BũG Ǫyy孒|t&ȁ1-Ag7:!c cƂ3Zp 7-hDž?2]#-v r$Mߒ9AZs"qiqVoﵙ[_{7rcCrIg`Ti`KvnsLeW{Kl`2W&{r|֡ղQ,v fU>J9IAutvS?gYrkGV0Jn:}oشρPo r m9]u?iW>38'yqiqN[;gZ ^IcCG9&޸sy3Eρ۽r]]dz辛۾}Y {_]E#M˛3_BΒd濯f d[BiKǾŹ}gFsIrLٮ;5Ό}2s {}Y0qc$}>8'Ikqcm&^#y--ν1ڞ禌-2˜ 5]ډmZ壛^ҩE1J)zkfN/|܌ɶ[noxfm)z @>=CΗ|A>Ѵ8vnPJ y^Y:!gc%0?;HvE&od_l[cKs1hqo G72lխ#Sݺx//zdW>G+z c=g+ [10fT-leT@lbAdo(yuP*Ƹ8Ǭnsm,) ;:i-Y/{?i8Y")|fn~ׅ82Ikcjq~ޛ8#^-z Jc%CI4en IDATާWE3;{\yR|Y$M=0.ٟUwMϿh:1r}k=Bsc$z-i ԶmvFyW#C -#iq'x]h|cL[c9{8;1bN=koGe9U(t^aу`tf ϯw}ͮT-z&`9R-rvEEr$yvS\@my}YVzz}^qmqnZ/-ikZ۞lYRz _6=$3?0[]t3;{sh⿳bc}!0:R-$^I*z`>{gfa%cr[P紵9Z )_88w;^GWlqB:QIgZBcC Ą`$\j {ܽmǥ9Ƹ8ǚ紵I>NER;cO}Kh6GPlG򚢇m W~WLsL[8yC`mVϩ?noy ͮӊ&]z$[:[}m3jq^Ij8A}tltrǃ8sb[?3qѐޯ>ۅ3~Y~۷5dbX寐t9&K/1=6¢@napxm?!霢& L+/⮢I)|?h nbJInk;1XswИi2PK[IƧ9sFnq"$IkHlO:N6_[Clm.X~zm7Ei1=z`0kafA`F91ۃǯ;PX;-\͟JEL0sO5}<erd[|إ7YMBcijZ:XQ8JOۦwkݞUG⼆<-iA՜nj0*-δ8jH.(3F-]o3Y'SKWJ9&\@oھ}溼9&Qho^.z Gxރ͟7g`;I-1F<̾v  pAPxǍ{Sws聿ksL*^\-_tns EϕtisLcV+jC`p;-A\,(γ}n<EуF_Y\$8Zk`8yqs̶#mqIs< $^sg}皤ޜecLwR)\rѳL(3 1/ cvfwc"~U\ey5ϻt~ѳ]S̿Z=GFwY.=>{gJ:Fs1iqϻϞc! ]TlZ^#}--=Ąsiqn/y/ 4=xݿuMsLf*/,zVv\)z Go*zlqjK\X;&{lm = l%钥OJZj=:awNRs̶q-΍%9v }mil磴8g:.-m#ڮ}Į Os\|Jh?ZIw<2wa`Z'g/|ec8JB'4s?[h{? v/u>Jsƴ8\-9|_ٿ仠~~tsh|/XHn9&/W?!LyinE1+W=33ҢϔP g|33֣%ғsosH:n[B7bzCc%+]} /8vIŹ}ZKs՟c.N9&Lkyv^2{sP咎=DŽl%{CCZtp썇Zs tG7$90[&q lqpU@)}m}~)`Ym9|:R1Fۂ- 8Zp.t&o yx-Ϊ+Ctw~$.Ssũ瀴sy3^W2}aJ)p''oYDѣz&!gIzϚ<>B7Bc1 ti,28'_@OQiGxs{ysL!\efIٚ%~?E\҉\mAfŹuco~\ ;ΝyyAjsۮDk}c6Lyvڵ5/KE1j?r4sc}}1{=Hу s\zI= G?Ed͛]zd O>I Fs>;?JIg6sn8ψ=<oL5mqN?%L܍y-1ְ_]6(cBNjcBЮ)zIcv7csL$_"<OrigCfǮYU(0NFr?vJKR9 5t6>i-(%#=]ۆvmq8wWM| SsAվv#{s>f=n92L]ǬM9?ojs|d2ͭcγ<~;E{r *w ^JѳL$Es=$\)zIEρd9~R,6/ܶg^,0.¼O6ݷ+djɫaMz9-tXJ o8!紀RRp*\=W1忱 ;λ|SԐsƬ1[9oĐsss<9NZ;I;W DZs9S>V_ٞpCc&?؄@۞z).핔# m mqNF㎰91t8-Ήg279\e9ص8 82ǎi;߱rWSV^-E1Ubsx6300E= &nւ5fcjEsZh,8A霣*},.}".Š mg_9I{LL3-ν!Z[2}'޿n|P:1[Q9&ԑMZyܽGofуlT3o[6=?|Aܶ={Q#>T P 97|tga+7#t){HCci9(%1Hhͯs^+c;λzq@{Nr؋?d Ķ?ŹL$_#)<dž GĜj |~UMf)ޱ4_0f+ tYsL"}/)ẑjy)zI>Eρ]4A/" 8KrCߢ"eȹვ+$iIBBc McK;WRܹX)yo?8t^EƱ9Y3 r 8w'iUs$'oqNl~~_鞳繪IOwkoڲl խgZoY&O;|O=F3[99&?v8XnYL~bDcrnio=]M&M (.4Jms[s38z`KO9yLwH[po >(NIw.I1}}&_QOgˆlk7r$A۽6Xc$z?c?NW{.Y[l9X9ujZ9&';p>ᙝpN˥%.z S3sjK-#A`Eȹon]*-f~xe[ێ}^=$rӋh2kʢ4&G9Z=nI[t0q]ȹGv|m4._tfg vݯ?Z~ӵ|~A&Ͻ^5fv?|ￒߐAs}v@3LCE1.m=Em߾R|NҦg0=G+ztvѼ$bfGA`uȹ;OYu=Q;yնChqn_yO#X;;\N 8|LwP?tt=z7[d=9t)z t^^w'iY E/:|= reܝ+KHG*TN%JD[wۅwG~;H\Uo`|ܫagiZdWLO/2%Iբg@K^8Y?fPӇxbTvL?1t'igѳ@F5c= rGrtl;͂K~ztw{#Y*es'};,U3 -TM&9&_:Eݕ N[MN/zq 첃_ 9r)A藸]&y5t Z-a?<ydǾW,(! KB Y, eJB)[iaJYN[fL(t tZh@ J(Y(e $ۺَؖY_I|Iw;e[ukW IDATض=Nq xnʖN8`7lmJѪݦ*;>'% c:4,oak]P++z$DT/֔(DDDDDDDDDDDDDDDDDDƒs?O38@hlS:)c,#aљ ,8~ 4Sq˓k фNt?R͉[L(n(s(>b:(DwM4%%a۹e./ZұeA%yK)Z 87y\S:x% eDfMEg"kO/ (F )o@/՟䈈(K\wU]mfC^ 8ݦJf%F :M皓r,'J;zBDewBDDDDDDDDDDDDDDDDD-4~+v-l޹迣;\.+t qLgʈ#Q@ܼư!} ΃e#\CPU 5g""ʦ"y9|*Hu7"߉ew#Xp5O/e9v,8Q_ϐxt"""""""""""""""""l$"SW@@Ћ2p) 8љ9 b?+ DDDc'".> A,>Dy9A7 ќ;tJsM6'g|PΔ DDDDDDDDDDDDDDDDDDIE&Wm:D> x `9Xp6OD),8?b1[ xwc87C"ʁmKm_pL;ا7]y(_h3ÇB(͕>uD*1^c:qکw BDDDDDDDDDDDDDDDDD4V,97'WXIu}4y$8ss76dȶ]w0w4S34yR.\p-c8Ѥ۝ vܰ]tZ/8t7B53Mg!"'|]JDDDDDDDDDDDDDDDDDGTRJbq?.ݭ{oiH%L] "ʞחPޖd -9{}m{/|QΈăa[8tz9YzZ1gܜv ~&V:(VBDgjٔ|Q$gª*Uw[r5yc 6tO\ۺZ:@rNqRֱLDDLRceHٱ҉&,&Xp9^yr\ǂ3ѐ2~hU'9S䩒L/1TLqwy̓]=+vh yqٵ''Lt‚LqZg-hXX)a'vC;e v~t\jWz0̩hӔ;t8*^OD4grjm DDDDDDDDDDDDDDDDDDpR.Y>B@XH('~`)7>R "ſ\mu_ gh 9C3SeW+"`3鿘wӝ(mh:hLX/*QLqb]t}TLu=47pv@DҲoՑ({g;8D@<MυˢMȅJ{6oz}S{sX]`,DDA/bNADDDDDDDDDDDDDDDDD|J~y֖O0dq`yɮC3MqGK>`SYp&"3 z>[Zm:HvL4Çڔ~&@FOF kDd55c:奿&z"""""""""""""""""*\,9Әlaɍgr7DC϶.*| ڻdЉ2Mq|8LDDE/D0ozsc29a,/oK8ty,9з ~u:=0t;E;ܽӁ[`(ә(?Dsǒ3YO\ t4%S-Y Qx"%e3UInz Y ;!""* nyPn:Ϥ,cmt8tQkJqA%sqQO,WM (pxT+5僪z`T,il3K4. \7h|s6rn޶M@{B17eԩ*z&'([8bgNp&""_ ;j9|GHSnQeGD^+E)tnEm[ S lYb'c: *z{( ݒ4عe!]yC%o7]a:3MP.S՛8g@ۭҍWc CۖRJ9JK8'@d)#O;߻gT𑅛/|&R^U˫'&_rjmޟjWzLn~)Yy1̵AJ9(g~zdvaHD ;vR@N穮jI7BDD4Q""Y JSR4ZV@@ƨ"8nt\ EM@Uhv{;'}6O&"b⺫"+c2Q6͐x3]Nsܐhr+|mAf]]ڂ :*0ԠHTө*tBD@2z6-CEm{kDy_;}Q&6AdIn~uî\g""L0*(=bs8crfϪI@ {S{﫱&M\ֺ\ D*p <``> SGr{ԏU 52M [y4c}F.gJo3ʒ(ٚaw^%Y[Zm( - X``5nlŋZBm f""ʊe^gb(YHү-^V`SMM'YPh VZ` U__ŠO;}UeXbvm-ϕQYּe-:\z8CΗEM^V`)=~(Xr7Y^+@DDD,ۛ1]2*YDV) 1ub4i?2FO$^W"Am*> KS DD/S:Ce P,p\uxS<ЧZm@DD,) @x@p G=+q8z=G!YTZY>[*Rg:5htxzr6g܊E8MPNP<.oN<"'O(O,UKB AsQ@TMMOr9'F᪲D!,p2Lz S>b=Ծc:&){Yf._^ntz>7sQuMpU 9Ǫo LqXqs];xX&zNÝw@DDD+lW7tQU#ľ sLJZȟLI/BY&xI Wχ>.6K5'O3[zWJ5+ Xz A (w=iDDQQQ}x0%'IәC-}<М}H`5JÑr-*rinnxt "ʮdn渚DQD ~oت@>\n~ֶDDD%gʺOWZ4zߠ3(8<䶹,|nQ\'v KL=9ڱZbsn/\:֓,Y(+PNw?b: MuBPf:eb[s{Q<)~cwTD(Tz>4)ɒ3)SD}4X]6Dc)=@> I<*4t "*UU*j)i)SLgRџ'] S,~z68t< z޻;Yt ";N`9{|N5mFDDLbM1˓ fQ?Mq41SoZS;8-hX""""ӪUvwIF9#Pֱ w= "0ozacV9JDĵXn: e/4Db[Pn At8QLcF9H$^bͶ$5+йg CB… ܳYT0usxcPh̀:=[0*N-QK?Ӂ'kvT( d-xҔLr'Ӗ\а,ш¡?C_mJ}tVE3[KU3į¡GE's(zYcLg!3"5Qn ^Sne:HD5%sdmJlxtBTQbF{ajUb?43 tƁ%gխ% zX3>{O) 뾝Zf:d կaBsTTT+L|5'ϰTTDya-wTInnjWa(MdII9 YɐhUE~L ߫4c )O L ET.VGt46;]4pKcn: )Σ Xek2)H>rnֿ """ȵCd>ө)]Q "NjtT0įNg9иDpKIBT ,}韅CD%C ׉t,kvm-IAH<:X8"4,9Hee|P= j\HJTޔL'pNVaBsRY^7= *Α *rNs[/O.h$>,6x ֺ߲ZajzvJRSdUZs9i"N4c:G.zӹ)9*+t"pm4Ưګ$.0tb-R3e:߈ r1p2%'EN~HlxtlkVXm2U9Y{ut,ēy3$_ U/ *8UY(,9ӤztUS)L<:c5cd~&5q>V9|MTqG DDD4*| s34%N7ዀhug,gUNq\а,јN+~t]V{1\'v7g iLY LY]M״$ۮ-)p""a;q_6HpWS[:1Aі+M0e,NrxX5Šw}%"UYY D\Tb|]/ ~t<~*9@n}9&[|? ' W[FUY['_p<~Sڧ|m}0a:'=xMj:dsVC<',ϵ|9c-*DJGſ8tڮoTwp6CTLfH`L3M 2#vG1r9/LUǢ seyy]o:),9J"vc[sL-ϺR}呿zMsҺO/D*]g_뇞; |{JRnةmP~YP)YocLF;asF8gv1n&8q8yM3Nq2g"_ah|kC|tQJۻEɍclN~VYjAcPLc;Yzs~~yhDDn:7K0٪ubuK"6qLEd;zEg`W_=b@TTqlYbΑm5 ]'vW\Ga[_ ~mDH(~~1Ml}9]7"#N^Og8,8yr{kGg,*u'v%U  \C/C'G4"bJy/*pXp(δ  EYW0߬`4Q (!˧FBs#N~.}K*܇X@LS1Y0k)#|PnQuc %6<s=skuGxaA9ׇG#Ql:DDDV;O4Ǯ,)G8xi,+@tN\s~m[;L£ژI'xt%q7 K犜krd eU 3Ѿ_SJy .*RT{a'v[$S ˰]:ǚS(DĊiDC#* |LȢH(zn<奈N'<'8yr&FitP{~ IJoStvzA- 7Sib<9y Lqr|V8cC~{'2MP@ msL$"\rК!9h^TmTձCh?V$;t%ŗMGV.O͇Y++kvV5U]bNn( X*.%ަp(!R4SLg *$z 5C!7K\D+=cwTTTO7&횥]QT~  bOtcĉG5#m/< l|$w@֓;vGe*i[ -zLQY&=bwFWh: QqU׉}1lO{MUniTƄz5\Y^n:P**+dDy*؅Cd4{l72l_z3bər&3֦eS-:m msms19Swm}9ŹzLq~faG6lw~;DX&1|rj(Ӊ=jFL;k`a0(4yLKt*۵)Q1U @< n(tk0`"Dr'Jy z%isMJ;vDzmQROk];p;tTIDc,9tvN[ 0DTpNNB{⇸Nn10NN8YrJ,[tYRbҿsT P"zXĉOyh3Jϰ4׎_'3QA*hK㗊,ʫIA wȅ"R=W7TubkP<И ϛ@_ʼ8˦✓))^r<8=\Nh OsL\,Џx&* ognf5TsPf Bũ9Y{Dxt7HM,~c%4C. 횥'v%hd5įRg^x(~'lOsĎ^!rә@UO5(Ͻ'jL+:S\'({4ʩ3̔*3ߦ8]fNqGLq8h?-l_fm"""nUb,"8rq(w#x6Ɯ`kS{-sPM'T _nI]<ҤjN>Vl1'J"Zp< LEB *8t&:ǺN7鉀`+e@p]ny=&.CWć]wU$d 6x^cc*U('v%6Yp,4?(] t EU٦3M*;>ub'>;8Q 3`ɶH(vSUղJ!zÉ GL硜xP?t$"V؉]}AC9 ]ijh@E,~kpM硜8-Yφh޾_ ";y3rUدB53M!67_Si'S;vGz|_hJV#N'!Xa&7f:Uf9hX;RWںa D;zN?;z[>r;A9CiLt<DVRܔo:ߥ'Rf:U9Yd҉?)I)W0IPyts%MG2¿Nj6'>tl E[ܦ1;Gx2q)9LIY7J\N+U - R;ꅣgOLS jkI>¿S8c<ȷXi: Z(njoO\;Oq An6PN1@1ۚ<*FWz?4i8əX$?imQOqVK=ܳ)8PPj<{7G:@@VA}jz}\7'(Cz ҳOzc{u+JP/oQތrw^rmu< ;R yM=}i*}oTv) "PQ)Dp>r,|wυuUσQm DDD4n Bt u"iny= \K^͂yzhXoxTu>}1Y.PT:* 6B?^u /8v+f-P܀g=}MYxun(z,D庫"P&<ɂ3偰 uK؉M,8Hy,/WqE*zXpT| XÁqG3yeo̙-STtmÔғCo?ڳm0ܓNe` =i_t2~hutpH{K=EuOIxsE⑊:+:+${WtNx!κowB,ߏ7ru<4/mnDDDX(NsvMvtg7euɟ΍CT $9hHkJ-_6ƶ۠G?& !(oY) o:D]"l ]t8GOxgiB".J;ѓ'b&cD>UUˌ];.[“@!+ONbY&jYLUn2B@v/QyseyPX/*Lg<x%N E4e"S]'vwk P ~s Ⱥ]WTq9LD-@kǮ׶"zd엨DT+\vk|I=/LWEP7Բ28ZA :=We!=VohAoYS,rp8EeM~ ;;w:tT.+,CWtج = [Wh-:k"{-PN`_tޢs (:,0-WѯνV9j~a{rz핝)Q&m9|FD[uF}"ZCvֿ@Cڮ)7wԿb:P-u:M8 IDATgr뮊AI3aٲvuxf"Q.|`y|VCAC rK-n>) QʱuxbYh)84}gh9*Vۿ l =Qxݿ* {}qAD+$e~NEddx_1øbLA'ӯ[mŹso[;&◑9˜otVoBgbNCg6v{e9k{yX,|͎CgC\ycǧs%gsz/ ߲&ެXBgClw:c:k!^&tNBgR""wK'%]|n(ph!Q=z1Zyӏ"" 6nίbmBZ )5CB;K = erte!hh6[e9Ǖk?63Ӹlܸ6c0Zm"^]!Zn޳/|nl"dJU?Y+7zn*  = 1řF;rУEuqf52Do =HD$ooWyh [2-ʶG!j49oV{Q !PFrQMmŵ̴ƕr ~PP0Enq^QxmqV{5/tZBg{NZYlkcC|rC=ؿ/[r{ ſ,jgu1}mgV9 y(Ȭ-!9Ypk.t:4%tήe91c ; 9˄vٴ 5Q4{~XMK|u>7?z~QTo]UnC0Uyƍ5jEuF_zE 砜?ƞC7GYw~гPJ/y%Zn|Vb3uCƽ,LܢG4>xu_w߭Z#z#2Q;xlгiNˍOtkܸxEg%/0џ'pL?.<7C)r)t--\,h Fg6:6:Gmf:CYMzM;GAٍgҿfs~ ui o_.^K&6Bltkl> % M:c6\.yĬK$mCg5|I6t$Yѹ umߓmu,-ùř( w C1f.U""W7CP0>y]=Q/=|!zZUh=n  E-v)m ~?\Ll)CBoH-@? A*Ϭϊ4Qm^N 9iZܼFDDHt*GJHÆ #g L@9[&fEhYbhQ/D!*Yg\yEj0:^ז.t6݄h ,t6{,t6& ժ.6Y@qeBg&tV/t$YEƤ"tvۚQ]las8tBswW^Z]烉hT}Vi@tD~lySIR䅡g K= ;z ^QU3pizq\tf.%rjySCgY4DS sj!S"MJ%@_Ι6%%{y]"bxݗBlVޣЫ}o yt5h{x`-,Ω_~o+ry7> o4.jY??ϡ~O0R?ƌ@ոZmK}P\4Fzu!z&|TK Pn0:[]}-dhй9 ]vB  }^9E.@.-våQsمBg,&fdqfq-j,t6Cg%r3ͦ, MmvǦ)γk7juG󇈉nτc(p̲<~lE탘|}nA)Mm7 = |{ӈFO;/8?"Z.?l4&3htɯnl}q_EYW-7~ Ja"Of[x?V8/wr[ :K>s33cFgu:BgKٵήk:1۸l7&s6VمVhE4,78 Ln֖^HlYΕEЫ sMHBg_܄o!ϸ_hTzݡ'"榥+opFOֽ#dEUgwLj) =Q?Dq)#gore hhM躧j-u=_Ŗ^ E4lq)w8;,DCj7[7] ._{vZnG7DDkE.^_os}/"IzKq㹳{ZZ>ނ@gĽNxӗ|#ٵx{g-Wc CB4cTnǍ'{QT7聈F"z/rxp?}֚z"ً4.6:5C4a79BgrzaUIcWR-ynNokxQs1pN70{†y Ad3Ρi:k:fCg#e::[%t4bvǘ~R/ 3}/niAi[TUkKj-cpYJ“p#z7iN5"Yp%o$Y4r6^MD}tLq<W  4h8 fhD"gU#UIx.P@^ D ViכCo~އB_TsW0RCB4D*z&Is 3hR3Oj=E6x+y|bLCۅ^9[;qas,YY%9 abd{R]بYl~-"C6Cۋfd?F5͢5KH4t3٦6bEPyld:L .b691 ^tl&6sbኇ//]me ""ط#zqnl}PעL xzÞ =fRǽos`7BA4{v|τcщKCACjkBC4jٸY+9zv$ hk&-F4" #%{:3$6S~qTT_X]w =Xm="7n/fh mƅ΀d^mn6"{r[u|Ac")MKxqBAD̳O H$O! a[!{SQsY!hhCSOYϝ-o! "Zy ̄hQ\ztf9^{ HD+pD׉x.Fvw|O`D=0 !rzAVFZmwsduq /DD]SGD.??L?ȕ#y@<ע2w9%bE/..8CgCv6:#ߒ MOu/gȾZkcL?Jh%硳blRrz-ɶD#;vju:t9V:g: [Bg1kKll)t %tv6);}ÊWԝw~w=ǘ9ZW̜ U 73C1w =QH{Oz7[B1T-yKx7 hT+}GL J:UVw1Jply;x>W3IFJ_>QЃAAۃh~~jON?[l Gqsy6:߯@:#ߢή)EBg͆Cg7i@j$aCc .t5@$ :{ۖ ۙQոM.~CgͶ2:M#eAgǶΦ:#lC3h$& :Å2sSYb3LDD4.,oc(~d&?zqsPnfygch">CzmbJѕe =|c8-46ˍ{Axݧ:衈M[g!""ZE@\a IDATh%DZyjέsTz=Q=pl>-"YOZ3rRWACR l՜;Ͻsz<Q/=by'g@Ox*гu:}1^-7#8 QxE!nq^ ma.ή7#dfakņЀGe[s!tvϛZbfBgY%tvՆmC,6g5mCg MK Cg53k.tֶ64Cgd3ZBguol(t&SKWc/C#b$hK4$"1 =Sw|{A}7;o =cDžB~x!Y _г}^ͧAtDTGzކ4[n>F i rU ZƍڽP] z *Eb.YIx2_C)V~P#eq*г-I{ViN8;i-zUo3b,٢Is$k)^06PN/蝓_:k(]G9yϑ_6/盜{=5]6/΅0%tǶ z%~z\otNڅƆyD;U湡 mVU|Z80n>i{m=W+ϽMD? t3Jס0P7"4Tzgwݿ(ksmR0Y ͮYņv,ZmRx-BŤ4ר:%tF7ˢٽbc6:6*BkwNLvtYtκ8tbi0:I:Cgt=V"DDD4,o 1)WfʅVzC:yM9 W~ Hc[hT'JB3ȩT'O?HDDkN(_pw>ݼ_/qѰyb|rU*T2T;hL7nG'n η9.:sijr{hh(^ЗDݻw = S C1f&{t!hx@mq/~DzjC ՛8;,DDD=0qzYHpbؑVNT?P~KsjB}WmN} D]&67E3CJN/(QL S8QO(p|<[XT^z""R9"j-(ѯd89p18q.rsw/}=,{C,.@:{4Ru r^ ~,7:`mA.Bg>+clhm7E Q 1n.:+ q>{a}l 3@9ۆEٝhqksڅ:k:OVY """&{υn3S"sxB9u{w]zafJs0ro}"2M!tFJ;',DDD=t^ !Z?bp!) mx1vy9Sl9&I:gk /:{P91Ga^ `o9z ='> h =Ѱ۷[wʛC1VTMR1hx o8'DTrFz33BBDDcWㅝz!fǴ>ojSܯoA3(DEqEmzeJ<#HD=%qcPH47> 9g!"":izF4TTe[;mt9~9 {Ͷ:-mh* ^aCgdϫ yƆkF:6lVmbj5&4t,t9 ]p6DBlcl#tĬjf/nNMBgM4 %;{F J֘~. =4c{nSP;zY ƒ}{a1ւ}=LC̄6Sг 7H8E6qmj!ѐ3`>."]Q@\=r/HD=!GH5Az""ƟC9iؠ8/Wyv{[Wr.nCg䡳Ms.tzUޅ>zjOEzq]whTJWalVz"~9 fJ&[z""""ZQT, D}12wh%_ЃzDzZ:=R9[n}zm=-+ fs ֍jC;ܱ6gs:?fF!t3C] Bg,nr:qsx1 s5 %tc|:盞x^,I9;d3څZ Ř7U/):qun= hd2kЃP#灓U;h-s )c"z"~u7ڛv_( =Xݵ_=&zqa4B@Dk""(,DDDDDD`>,Ҝ =(Ѓ4'}"""53 x}|9t[ 㴹 8w{6:kKlrR/Dnt: mqV39:#{ͭ桳Q2Æ&CgYabVsM\:#,.Jvss91P50jš%t..g""Ѳg~秠z.;;J&yLƏ-Dkݞ;%wB1De. DDVn@z"""""%)άU ""qNmC0%BBDDD F4z}v8w86 W5t.ntN#eYypQBg{q[na{lHw 5Z դ)ݸ(tΎOK!t.Ë[CgU1\/tV|wn.t6ec 5X6&4(t.Fxo$'""LUT^hY"7?CAFoh4ȻBO0&TmWꕹ+!xM9J“BADDDA[!r'/;_P = Q9.6vq1tj"߉:2Ϊ镲vBtECg;ǝ6: ͰV:A!tzDPml̢m0 MqjC,t6PM~9}lv MY {op3hs{CAԉBA,!8/ zkFŞ_z"^0=E =Q3%$=[Cʜ&Mz"""`Lmmqn1;wֽn3jy]Xlm 5 :Q&i9ʶ/nq^"t9߰ {(tv6gY՘f<|s"]ݦe=^:(QHb ICg|:/,% (KTo!v0>sڿoBA4*T@23t^&uh֦;DDD42^1nzꎈOxxYV3 @8/ιg\:0X+ZBgYEٵ,:^OY05\_4Z)ncsz1ifq{a[ -6lFCzM oskZBg${i:d[PMs~/p3Ѩ޺ ^zEݽcDe{Ɗ[50"ꅙßpW9F^d =כ5\`SYVa- ""1^!;ոV>9DDDDșװlq^Alqnb/M1n\ s.tF:%t6Cg'Y ۢ1tv Mp(Κ6fǵ.T6;FycT: :LCgCg]:kIO#Inǟ'""5o_=)&QF΃w^zQA|*#O"͉cO@pгZyBADDD#ʍ3zZZ^z"""bLC}`9*[W^nrq%sURxmQ~dﺶζC܅m,Y<,6Jvnivvq3X9n[vKCgtr1t]tlw穑~_1hy1 wH'm!FվC77cԩ3Tm@z"""""=Su!DL4Mps2=f~z"o9RKгHz"""SCAEʇ?&x1r#]mc;nqF}Wyk,y)?.ntVueQD ݖg Ӈh5kKBsi,cdg5$ Ƙš4hn Q:38QNuDZ s/6vmBAZ);J h>W1hH4'jqBBDDDDDKyֳFs= jU\z"""^B@ԭ8/ey[JIŹ4QXc>L4hvJ\ skOۦ@U^]\0_U5j4(TMdǧQݥQ6:q?t{a7 yQ9Qb7>w M{Q:C0[1"<66X,t60&3`5Δ.,ήeCg56Vj r3YZx%_Ƌ[4i Уh GC@DDD4NӍ y}9G1nz """ .F/uGOCAIwVlsƁu$z"""""{`a ""șgbˍǩBADDD/ibtsw/}nqjsKntV5SFg{:]n[,hBgFɶ7W0mCgPb .h Cghq͒B4n6u5k~k(څh G*P5i j71ÅyFYݽ]cl!3BADDDDD@z"""!b{>9gFpjr{m[p?6+⼔AlqYŅHCkBeb9sBtd7P("o#t9e1"WRDj_WhSB" *!!P#@dY1v(_8-@z( $a(JI)mMwT" %k 9:ř۷P:p = a?ڿCA}` =(iNU1(m8-raG*wb"KuCw}DzlFM&tHYYP YPI-B3?6ρ. %bDGuQ>FIk1)D g)p&BcJ*[k[s = Z}Q?*c@~@Nݪ2 "'0%ELYpQ* ߈WGu'3-%IʦTM^)+)EwEQCADDD4iHufʃ`nqn5 %{iv8Yb^lllg2u^7&i, nu} 'YX9 4I1ZHmgK7LG6jVD5j4t'42iXlR46D 桳-~ 0d)FPoJqG桳-j?t6YLDDD+o*ͿCBkpBkLTo=(7Oh|Po޽cϠn|®$Os6gY6j (6=7e̜b<3;GV~Z) xRۺkzEx!* TE<"f +Jբ(^{LI2;7IJNd53y4Q}S%gU..x\˟gO762BUxrP_oTl4qa!ڵ=,9w 1z%gC]qc6ӵ|_k֕bX:6EgEgm,:KPRXk;ZtztxDEgDE(H\/:rƢs/#*7;ъr(XY/:#(:prBayi-Vm!""ԛ`V@kJ=_Akƒswڐ:Q<}?9h`TP*#M 0lmD""ZbZ_x_,Ut WIUD&޲e@^h`~X9vJɻaz>xb^ "ѓW: (27aC$#/tgc|Po8_Z码1ԠXM*lߜ KeŹ!ъ"v;+EV~9,A3mZ/:kӢ3 KԈaɺ^VcLpNr/VtV (Ex9oH9k/)e茨,N^@G \d?Pmg"""Zr9{ȎYh`< ^Gj۶'l6:GoDDDDnjL<Z砾q~z]Mk=uw}";37 ȶ2z=@CC_,FD 7)􂡍G.Jfg\-#; >Ʊhpܤ ֍kôR9";߱e߉dxȜAOuU-[iv7uՓ88 `Cu%w0gsfsF܋y;F*fX3pu$2˼)np@~ů 2=Wsm۞EgkzS=N ߇C;Ò3 48mOWzZ~p3~Eg+@GEgQxxAѹ^PU8֋%h%ðIrpU$VttPPXtƇTefIRH, MA9H Egy)\q&""J3`l<: ?x9hmf6<#%xc#ӝVy)=~Qn ^~m7G <|qQgr /p1;8:CA:+],rF688E`əh^+s\ȟm7 3Q`s[B'7sY k}ɉOo>"Ou?ABƣ.?˘˼n?@NSW m۞Ed99g WK۷~vx ͫ<%'{鶑ߛtBmO}|DD<8 8/7^8/_P(謋5g"\f (,W=g=٩#_' r3P/:+{ =Zztsu gsz q~<_g"""ZB%|t'yu_"F&!{xV5!HF]9 -޷J3뻱M0o؝*xSdnN[o2r垸@Ld o)V'?Bao?o~|:6Rݸfdʓ-*6 ]=if/^/Ӡd VhXr>ե%˛˸_8ofŹ͋U%f VesXt}HpBh\>,:f"#hP|` Y<~"uDDDDI2Oʂh" 3. ^Da6笭&;<qለΓ*tK2ymOx솋 <{\Uwi[km{cj> o L饢>zVo:Lݲz/8:|&{ZGtnwu~1?Γf*xXr^{>A]Q2kPk8o˖{y5w/K48:ԱXq^ 8/ZᦢsTܸ&:6V8GU,87TW \ _tz4P/J ^lΐs8 +:G ,Rtvo'A5`EZ=Q΃sY/cvQ,R3L7L]Ng(nW[sPG J[F'&T xو,(;^(O^e$󯶌/C=SܥAz:c7XI+':amgv笃Bta96PC=sX$_?Az:glΓZ>X9 m T-ݮfBt'~y^xhDD4<D ފj׽_q^* KZn<6,:W5t&j>#vbr!AY5:6\qb34(ZCs /Kǝ#u.tjq u~^W ccW[hDDDDbyb(v[~{~`:$:Q"u`u /Eգ+WSTpSUW(UL[?|?Q= -ֹzRp^Z+Us 1,7y\!%琪j:9_b@opCsBG>&-b>EY'+(8T+YRK{<19#"˹NKY~&{K2b5P&"/~ˬP'ʫUY'R(\yns * !Qo'R@U\yg &#"J84s*0< A6B|2D}Ʀ2 +^q^EY::GsAt+Kr9Z8/Y^b9~Y!~8,.7W_d2/(:͈v~D_XtX~7*:[~6i*:Eg MYKC)s<%X<7!gsw8w֠ ݑ|O $sV`:{[Y5Wlpu." ɯZy˓ ܧFvTEp r$:Qby TՉ7nߨX硎䫹b53|:Kڈ*K+4)KϷAvkR4rMOn 3ED4Bg6NR.|dw+Œ3EeEV۝cvZqnw/\b9yzYרDfi.:^tEGEg)[hYKx^Âs1Wc<6aYyH# IDATUF-hqG9=o<: X=?L#o`u!( ?_?^72s ښfX:ˠ ^M::*z.EDZ'n#Qd:b^E^\,gY)V'0>:K(v֔T%O*:KPUx؈> PGzu~R>=>x {muAuHT]|3ە+~ P bjnR`?U3?ٛ}SOh@-]Rqy-Eg "+QVجbsѢXY'3V5Zo|ꜿ0PtFS9xk*:;('A@ вtg"""ZuئW귭u3)J?l # OԛBשfgCe`QJY _~]oo'91֙V^BRs[l_߿lsPb%{u~T=>A^j%Eipu./ɧ ûКs_ҏ[F>ϳΒ"޺): @aodfe玍L|DD? ֙9Tܺ4:Kԧlyc,k9~lVtpuYEfѰ,0,2׋Ψ3t ŋaYegxfzsl!څ,4\Ugq/:u["""@莯K,dRu{!8:CHDDD(*+7 뮜h&9h*TT'$9珍x̯D4XnW[iՇGOF<*5Q}n:U X=td,9S&AU-#wz"YЪUمJoN߂qrKGOFf~Y>πr̙,w^ND4>z Uef|[{OO_{:o]qny9+΍zX͛t@^d&]"|eYpU|99  B@D栺K0{l2WJ3i.8,9S_sӡK8(bg?.5oOÕۅ$_a8,:ry `/5Gb|- ۅʮIjx?Q_-_ฅEvx?&_ """jp5w;fzKoTe~lv5ADDDr7Tͨ? eYPHae.P͝/O~J֡k4`ҳ⼚sˢW%(,G`]?aX*f4 Q9[(' 3yV*^Sʃ++ցK*Eo; +a1Kl9^tn삏IXt"cZ/.Ņwש6㗣kTtrP"^bJz(bEg6Ț¯vF)LU:uI=~{DYr&""?eo)Varr o@lݸdAj4bqTk .QL2!ٺsgˋTZT9@ @|,_}: ӡg~w`Ns7U{c# [ ""^i53Ki{ZBeB=-YiNabuԷLW`+-6sG/:kCn0o.Egm*:z 5*Zfx9FcQݧsXn x[^lGe^u2oA:C9u"""+==P|8x4#gZgE] SBr\qScRjU!LzujIE̮[ bޟ9R1Bu!]o 9%T*{2 3os$-lxA^.wBe .3tWZg!"Zby޼uA 8mOW>K8/ش[q^jx,/+EgmUt&}>,:E憢/VAx$p$Z,y+1kx9u0뗈RÀ:uw:u'ruD3QPLJSսYǡ=~PujyGy^YCgvnBDTq_ߩA r׬C$5:G D(Nv^q:H|@:-BÅr1@ƣ:GE+Z{uZ0buP{ՇcLBD9X =7XD,9S_e]֊Ƕ./~EXq^* [JS[+.#N4TtGAw5-:r .G:DŽEg Jbs69 ҮqnKDDD<g:LbeNqXGH2dXr&""wS&=1_}::d0l"*|%v.\ L= I,D.rviv9J |:-p˺:Dfr_9L!9?:GRJCPuj"y91XE:G=:@?:f Y9(FKGw\;(<׈Yf爈ԛ(S[T-/K](/@WYcαŎ5(,KXâ9q^uc~9aK.LicċEg0 /-G%fyrDs3xr9,9 y5<:C X ""J57W>ZGٙQ?Z=X]bVP؛/Ό</Xg!P.L^`#iuQEQ^v@\ Uum9nӶ:JpyP>Lj|u"J9ќcJZGt,9Sj⼶u+hS]xŹVW~W[ֹx,DsQ,(:YlҲFݽL.:U*xa:(AGha좕f2DE%^tv9%Mg"""ꞩ*uˊwCP(Co@DDdG-Vr -#9.ϓ*{nBzőb%~/nZ{6(zu (P(O^e# 7u$sZ{ueSLg4rI4=}_ALUXHR/:GQ> s@E/Y)TwE (QJ \x($Kg[K|k+Kz\Nʊjצ+MU׬_,D; E(qxQ9: αҲyTEg.ֆsc9\`/\SE \߸h9*:k=[q&"""ZMjlF:s5kTB^lh-stF vj5*8: %+uRd.@ڛC$*2r3gPe䇭Pggv?PBD"+UrQCXrЉR^3֦n9XÕ;Ǝ ck x ƍEpu͗E砸/KTt^5{\q&""SW޿X砵Q{3[sP%9h(TgByϭs?;0YsTUB}Y(~]C$Y~b=Sս\""Gs$ H4YR{0g/_YH;Q\m#ɆuSs$7PTP͓T}[g!tPя*&uKǸZV+\t*/(:_t"_V+*:kѹo'GEghl9%^tWr 348u3usYâY׶"9h~]!ǜc!p:Qꈼdy:G&rj. jMP|:%\>Qu;o"J!j#`h9:s9>לVwIwNΑX|nزmT_P.BQL^)Y(>Uy- Νǒ3+_EvW[>Vmst7yαB2ⳄEgѦpA):x 7U]1QuFp ZzYҲ Pt/7 K>\q&""np3Tc))$2f!Du""di¨RqS |ř|:%WCAm(s1pkFT}@ei "ZgH,䷭s]f#ޥz]#|nΑb jKAߗ N0c;Ū`ə >(+-nۯ+-/Vnܰ,~XkmXhYQt/I $Egk2"sXjn[t spEWJ~f׭ a`(besQ幃Rb>Ae<:Gy̮[PC,D"o zT09RŒ!ҢP:G)` DKQg[gH`{9R7j"iQ;s'RMͅ$ %ӓYsY(YNubəW뗹H'\n#{d}!D~zDҚBΑPamruR`@G7D&Nxk$Kԇ:xy3e+K=V˹++*^wtŹp9wi?oףו$:^t"(:**.VtFpYcEfDhqEg ҫiQxT I@ٽ7[ 3|R2,9u[.gB'm( PL^ozOƣ:|u"L ]ҦH }:Fڨ73$[g jE蒡mcCXy\pʄB89F4TYp'WSIwǡ>ԊOQLӊRǮa9ԊO+%9pȈ@%(W^jV=MpZ'^4 XY%g F%*UEAW8'@=,^Їn+PzPHp[Z~ZKDDDLݹ7c,֭#2n#%$ccQ3eOP7#y[7(ΑF }G:yuS,?Cv{Y!R55+y|tǯo%-'*ZHRc#(oy^fuOg'CMG#R~YB~^YΑ6ãG;W: `uq6wZ0i!HZQ>Ŋ3{26x(c9QLV*9Wۯ8/o!ɪW^mXZ~OPp=f~Ysp_N} x/:E9 Pt:(~{6i >Z[xYgH!gBN_՚u"< Puם9҂/ԣսs,U]]y ѴR+zyWNI.Qi9| HXt ~8v mk<Tt2ᚊή{gEAB_>'""68oAKѷn:1nlp:d!ɆkG3Yqəc~CP E`=Msg*k,D߆$) iV*]}-9O1e(WE 96DM}:DXHÅjP\rVъsc{cЊs _/:^lysPtF ,4GEXys/:EqQ9\tv~Y_lB_o""""궚ȇZe9sd`ɹ6LD'P:N!LOnAݣӬ3^YwԿ yw:m(n8{C |򹅈ZH+ C`(Tܠ;sQ?ʓޟtyY/_~\EբF7 7Eg =HCix֋qs\ ~FE8uh(:wFfiH;p8Q7,lVNGw:fyX ""7|R>b/2_}: 쬨 |g]"j􃃕k~gwt7}Wƣ!@F D=)Pu$S|uMs 9Rb*4XJC 3DԟT| iŒ31祎ŊR6Ǧr9؟Yt֯P,"J+GEg /5AӢ]|:6a[ 4µsEtXtn(mGEjVnӦS[sl9uuapɹG""n~jbu_pp*T )1: Ln[ >"u䛚(9JD/@bqߔwYHuB -D.I% >;RIs xBS4XTo<*/DݥC_V,9S_Kʊjצ+-փ\SyaJzshJyEؠ|XEy9VTFp s`q:86/:/|LDDD0?`:ɶL[’spə'3(9Dc9P<:G(^Vyu[ P|:93$a;!NDi!Q#GceŹA]qn/0׋h*:$("pxYp~i9:8((G@s Eg׉cʴ,3uזSAkW'9w%&V#"02(1*\fKS3csP|4"[s7Yů#$A1걈I%g_gvjb-|oi{n!ul#%To:O6/Cob:Z]l!"K[\q^\qGz 3ɍE簴ui좮 ,:ℷ ~[}Vpְ,3\CyAScg"""Z#ODZg<"Ĝh:CDD9tU Ͳ4pzu47E`V' "q}Hquj"C:Ϧ c8s.s~#qtT @em'Αt |P"hMQ(ޛC_L}+ˮXNJ+K,u]X4Ҵ!{EgKyhl9#^tEyEXuEh߉_"""%lxm:c#':u[oQoA<:p/T': >slDdKUvYg&"R7H}FHgS_Px|n-u﹅ }9:%Ö:S(_`%gK]qnq~]qnyۮ8/vVE6E  hZtzהCzA5]_³"̡|s /KptǪׯ[DDDD1[MDnAݲe^)s@e:i:(܇7YgHŵhJCXEzniIG"3_XdQ?X "K!i@iSiz"Ƕ_qngpű-KK A98$ P@=Ċᢳ65:Pl#jP6c Q ᠪ1s״JQ" g I牸 ?oJYլ3$XDnuJC:O#$bucP򨪊껭sQ93By""Rb(AT%>t5ZH5?ӠRyuseεATzVw9Ȋ~:ԯ\qng, WWS6]qn[[q^>E6ESp]Ptآ_tF3%P^K=K8/뗈jldKo:3 87zctyaѹ4(beB#DYUEXrԊ mӴԱM6Ux9y~Eg ef/VtF+:Eέtϗ+DDDF27NA=uWSJu9+3:% >hT4gSK]N>S׭([ cjꪢuZ@wΩL:-NTH_^q:\rC|<BUܢqݶcb[7|8QI%@qD["> cə\8m8_87 EgEg/Xt^tr\0btp/,5h^,?/rř@Dĩ8!,d waɹKƒ3ʔup_r$ɣ3$~fK=B=QZsF*/[t(V&pu"bY K4@ܘcck%^tE(s#Zt 3Ϊ᧩Q9^toP~^iX-זχvdc#^f@CKDr_Fk#\qN ##$uguP@UfZS#BE~&¯uٻ8Tt>@V/dzz/\TAAE@ދ",$3 DbGd @g:ڻS|TPs2l&ZW4kq xк}8uXm^oAgΠPi_t^o[;輶y?-]s/m?wY@:9?r0]JK]:!vDGHEg`KDINg]΅UJ|i梁[̒oFg@}n:@Wc ~f -RE- cȹGn9xt(ۏ]s\ al|86\gq#ƟIkMѵqWέ׮:/=HjWa닕WKniՠkښS_@`itnoE4>DyãSLwDGȫ$i BRat 7'Ɛs{Otk-K9٩ y`3sȇ;~{D@}Fsl;,[dX>1&׽sF@OApB0ys;wN n 5;lm-Z~A-΍vZ9Ŕqm([Akq^fC 7nnrju87xк&6-[YA5߃-m4Łd[^nyqAXim5ҋ-Znwm 3ULGE@&Ĉ~#:ŹQgG|:6åo]΁KisXt+Pt=Ősƥ-v$,M9ׁ0w[dM]@ƙ1Џ'\͟({+9}ߏU 9#c*Uiq|mQwu m8d XתҜ8=ܲV,836:n~yi/2 R{A3}&;ve.3;G晋!޹kt_0Lӓr))}<:}7r31`mИYsȃxoqn%ȸTh=#.g CȶuCT>F}}jq-^V[t!DkkWgW˃뇚Wv$Fg-G28-Xݢs &:: ɹN۾l}=:z%n뾣n!:]{s >`:i[:S @p1nbEFs -3{Xt\bFGCƇg CȰUiqRs38l_t^w^bAD%ɽrW:/YTKξu7@aMm^~':ϤL왉΁1O@!0{;FEHtqyrrʔ0bYOKr }ij'?6w>+xta}YÐ3-iqnu絏}emfk]lbN4:/=Wktey ʠgJ WH@1M7Gg@JiqOFTIrr "̝A$@}_<|}P{9^ IDAT˸!crFFuŹ&C8DAƯU͵MQn +69|:^t^nkꏿ֠sڲ@qLy弩gE@熔!>@ANt D 7@E:;:CJ|psǯD}5:ćC@*`⃧1ogdfEg&>aԍfklB8/=mqp˪F64/?jtZξ:lּRڼ4ܼ<輾͙g:K΁f8}D@g'%#rPHnƐ36Nz`C_,|t%? :C{ :'ӳSƢsȆTEgO!crFeŹ纃ִ8׷ڃū-̒/:ZF祧V֭i}@~3 &G_(׽s` M,kC3{D+I@4I-=rd=aI]o<X7E'}iEǣ3`ƹ$}bp[,Y*HR8 䔥'5 9#{2\k\-5vq8~c$5kWWbL+-&_9MVץk7ƟIv~ /ɞM4=>;::vKt@=|z~1͸cXs&q#XYp+@% gdt( C(KyoIS>>q÷CsRy MÐ3-Ϋs\jv-Ͷy5+V֋]t^mm^S~yi¹| -Vj7I΁fiW 9vsO.}7:Kb ō`]gX M@~HS\>HN}B0'&vݽ׸ ω>1L;Le6Z7672-~&5Z[np ELjL052hirC#)͜!0ӝ3M_,;th+Y8p!:pR[Bj%u G\ƵnkȚEu?5!g 熍-is õu q -΍tZ4nqnIk3߫Ͷfy<謪A]J=bʩ5rwfmQo΁pN+_tVtnNgf郣3@?73`%@\QvDGșʑ#X˥oFg]C m(?N- !ȗk]guDd;o !rFdŹٶ-8^ۼiq[Ud[tÁD-΍pkQ{kdyyե΋̾Jk ?KZVȹ-tnti|AE@{܍Ab@5gf9]r,2ćysǣ3`EP>\ R+f 9w ))$?܇A 9c46mB#ZFZwj8q\Zaɚg7 :kuhyi垮;IkW =Sn/΁2̞h#C@?%ΐ36h5)OYvM>ȝ-[N^ `]#ŐXsN7CoEgֲ- 9yc{ d A}{ [_rskuXU6:$W :N;[u-ʠ6K/:ܸѐ$MD@>Yjo19-:ɐso=ҭ!_fx*~X'i27dOߋ{5Bj m{ !~H{ O̹M÷E֚Is*mgCȸf-h}-Ν ^87}ʶ⼼AgS*vy}ݗ}FYt^_ˍuml r=1:˥O?7:ZsgK:#ǶLdC{oFsA$4]t]EȜTAtc~b0DG} -|qv7+ `@vtKt]d 9gCȘת bPZŹ뚍^5+?*_`vU :ki9?Wkglv(5H#/;|Y1вo郣@?k0R&`xt1 @!U y|'+> bFs=8\jg-kܾ2t8^\{Wڞ}eyi.)Y?l˃p<ȷ=[s !"o.Ft > 2!oEg$ڪ'ce?x9b>,b֏ ķ8n n6w-^V\Ag_tş-5:/:Agjta/9-#pE@?qrlgcnߊkK0,ɹR7lanTs߻ itrFV@! b:ŹPo[-Rshqp :k4:絃VݵT{h#3ȟ$~(ҭ9И[ Ew=/!n+etr*KG Jj|z1*Rs8Ź[nn9䋃K0/yee\t<:PS\ q87m6/-Ν ^^SZt8\YU_8a]G _.ii^~W~yt友,n- 59yv_ݏ̿!L9W\ IZN+Ɛ3ROö4v%!2*%M1=hqް~8w.\oۖ[ 6jq| ت^NŹZivWAgwSdL 9tՒNHIo6tktfDg;[WhrвDƍ` "t*: ɹ*r}T ׶rF5jPo[TL\Źƃ0q osת[ȗ{W5ŭlK @LzX'&~7:jsO!/Rr1:F.;NF'u$bdf 9@1lY9#D|mQwu miu9&UykqnBp.ǯ]7xD5?GZ33)Rב58=2{N9c&s@|cqM&&0d)F0{µ.$|')cF87&#ZVH-Źlym٫}ou_oќggjt橒actlt`I-:Gm]uE8?s-]$&C Z -}iqdg.I<nyRos nǎM@àg6 ,30Jt<1ptwk]6%!~=Ɛ3G-Uiqn븴88FkQwKs̋+O+f-nՃεlf+\stSzX7Fg(GM% 9hɹp~ L[eC "ROO1iimPUyzyf:輶lqӥ;%[fZ7ls-Dٟ{oȥ!NMBt&Btwt2 M] 9#Js 9c87m6/-Ν ^8wq8-6:ʠ׺AzyhqH*- ¤?GtJ\49Y E_?3`I 9)2S N$\6rU^J87rFfZY[sMm wVZjsm ǯ:otʓ+ξf_83 ,~L96x?UwI7G?{6;M`q#XMDrOY&.$!2Ҕsz!g ޴8WZZiqnwm>nbڵs+MAF紳kI0ΰ13>:.=nbts`  |b!qTǣ3ttOȑ|tI5ōkk̯!hqnthqZ5X3ŹٶZݠkD:񋧜W5:rvYwr_"@'fۢs`;C?3:tAŇS͏Fgș٥[C597 nӾc2s"^c -As}mqnyкq7Z4kOnέ.dx57 䶙{&sF9j}Gl=/:lҿ}@t!;8J͹~rM59Ymqm6knm8w,[,mE̝8rs#]:]run̪8-䎕 I46a.OtH^eIs )n#LHt٢GgY"i{tE.qmi rFŹζ-8^[x16.pVZW{EnqW]._yiq(3Sңs]5)%oݤD(~L{htTj`r+e6: I[s]t,:@ĵmdIDgcSYUzz:n}>~sZZ@WgiqαGOscylt Hn>:CAL(btPehЧCLtm:;:Pmȍ7䉹s 䲻Dg`r`pv6_rskuhsnS_tf2Ox՟; #-/t@7-gڣկ-_]V =S;s`u :zfLpt?y0:VnJ0)Ǥrtqz!gdOUGs'Enq^vPZlfR/ @^M\xO%ߏ]OO@Kܱ8Y3Lzܑ>Ó׺ƥEg0|~,DfdP~y9sg!q?9 @m⼂5;Ź1\qWA3EIMEzLLot"sTks%/Ty1bLƐ3>| h>j9cwD'ș-[uZGFGmL:Ð3-έkŹuZV Ź7ijl&@mI*難CIP'@EgșG={*`t\q;7:Pe 9 _ir2/O+\>`g[k77^:j帴8kx}fm-9>xtyCY)-}":C$ ??:WccɤGGs]4`u CU ׶d))n|'CȸfŹqlm?Z71gO &˧\QPt,G(ۏ]sI_Q&ũ{D ϐ4#O?@.8:@ޤ'Eg-{@7\2wm#[g!g Ziqyk;mqZuܖZ80(vgϖEn 9 Gc/Qd dX~?:7f;Mzftx|":\%:@ޘAeG&$M@Τ0:CkqN@?Dd8wqPg5 n#8Z-Q-*Qht"r+vS\IitsCUREg({%:'#?/b~áC{o`d,+3:m%|gg6-: Ig$r`fklg-~&ZQm[1L\'Dm@=rBt2. .+:OlI*E5wN [l=iC3Nt4`t49C?)mwD2: IC"Dxh0q:nq3i)߇uvemvkkR{St#>bC/P`"x<E>HٻG`j|fGChre;!IrcsgH:#oDVXx@0䌁f۶;TעFk;kqF8FAiqcj4}$>2ƼO9r$6(Oq6;wk!\؋#Ѝs}':|Hi辡Gw.@KvvEG$MFg`Ns-5"نomjZYs#mqZ5xL326v%9d;0L=9r蟢_Fg%@Vs}5cf\SD(3K$D`b>a:8YsصARnr4`M^FKnḴ8 NTdtc[^_ =!?w3c-~1Q0_3WFgȣJ~0:8xNߗt2:Gޘ|6: !?#Kw}(?-i::CӖ5nYs}lm-uz:nZG& D &{Vh/rc"H[Jnog3 #DSc/tqt5:p_EsK=fǢsvGz*':FB/ IDATO#א)u--Ź/-Ν ^8wa8ͷ4e+Ǡ{\M:-I.TC$,suYt2AI9_w,G\#\ r>5>2:kj{J_tgR}':B (6Os!gdL:"`Z:niqްfŹV[2Ȁ~SOG(JbɳݽGw!2{ӎQ$vaIQ@{nt`LzD!JrԒFg@q)K~!PLoP$ 9#S\cO_YUhq8hPo8v@Zeܷ;9 ϗ[RIN=̃άo}It m3 O`^#>7woG? yd3΁br!{Zi7MhօKC*[/VrF֜֗fC yqhsKiqn9-Βd47k$QPҒ2^BfoErp?"#$%+3ĩ/tVtȃԹCu( *PD 9#;*hŹacp#0 [wص٠jmŹaעqsL 6X[0dcs# Z>t[ȓ;@٩WJ:=:GA[jƎk&G7֙9c{~*:GQx))Q\ !KJ+i4:K^]{Go.Bt21S#3΁32ףs2>䋒sŐ*M1g!ZmRs͛v%^u VkŹ-΍&pkFz\kCZ*ݧ`Ϟr.1~[В+͌ƨ>;rQ`JJ8D*/s3:97&Gҟtvt_n09fvV==:C Q W=]w惪G-mwCskiqnz篙d;`fzz,Ƕ;wQ7L`bt既s'[:\@鑙].;p/F'h%_5fώC<{ĥ;Oȷ'K:#:C߹Ϸڏ6_rskuhsܸͶ8/>0 @LfsWn$}EczTt"8zG(ߜ@Gች[R):KNnKZt;m55:Ӥ s!:C]efύEƐ32{Ҥb`ӭ׆87}ʶ\g۾8ob8~zAmt/59>8{o w^nisfn_tv|}ΊΒkFP.FzPt䕿8:[8:i1~::C^x$k-ΫJs8gŹvܳ /3.9 "+Ʃ-sGƤQ?{zl }`>tJtKR{ @L yg}m78Lg3ܽ&Fȟ]@7I:#RQtSIrF6]Xhq}Z~sתZZklgd 64>>s??':G#~JrI'z}SWYzDg_=>Lz9"yKRJ@$I yg/23~O*yyI_ΑkىGD@L^9%#2䏬7ۚڍŹkqiq--Ǜ6B6Ru -Kq|M 15)9̯I'Og8:K#{f9 ⓇztrȾH#75뗣C ?\OrC<{,|;L`p}s7*ŹPo[-Rshqp|^Z;nvn鑭*\&٣sСxC[[u<,rNM]8#r1 /@/O5:KDgP<5:G޹۫βrOg012$IȀd|fkiqKs'Enq^G?v}im-8veEeWk w!I/1!I:N%|f0ٛ&G<&:M;ʳgWc}4:F;sC`M~KsJ jqt=}lfm1+s1xOhʟhq-m&[M=n0|mt؋%-:GQo<kcg+-\CJADf(Nw=5:D-$[$΁gzɯ{T۷_2i [>.Y ?y0:r߻`9 љKc`l/_xW^Ȗ!XH*iIE@0XnO[8+-ͶmAկw>jUh8Rs#Zn#87[y ҩkcczt.{Lt2s}':ҠEML̞#6etςUnfjɫ$=4:K+CܯP&;$:KJWul^I 9 ɱ=2=q&{\tz 9#̗{H>L-5"نomjZYs#Z<-u_iqmxm`vV7S$`뱭!IKv$Et4}]752s @+.NIߢ_vF+vG?qht!08fbd<(M9ӬYT-uz:nZG&ɐ6v Q9ǤߏQT&c#ًsPb̸.C%K^%)΁ Im+:Pr\RP8tSߌ.sH 4bf[/%MFge\#7Tyȶ٧J9q3/BرO}F[ŹZuhs;iq^Yߝ}ڶ6vT̮4RW鹋7g}-ߢs̯Fȳ;_MwUrĞF֚}IvK/ l- -:kwڛS.W{o=-:DU II% $M\4Lѽ7Fv#9 ω=+9sw7T>x?r?J"@1䌾%'VyU?Z 6XKsm85=Of-oPtfmQoQ`R:27:GmO*yv_siOSe6;4U}!i8:O - tut1ӟn/^166{RRt`Gg(>\ȖC-[҃sc}~{e/Z} -͆zKs'ס--fŹI8W}z.Ɓ(ϓEn/=th9)F(_,>,:Dy/u IW WMU2AtsP}(:C u g6m]|Z4y5D(ߜ}jtdgq}u-.PtڞqFtܥ&4:_GW^Fz\G_[[npܦ- 64kOnM7 vZ1\96PfR+獠+Ctuuӧst)-It<+%΁nGv_avޖ0ȇљK}I,Xe;p/Ff,|PEeߙyqtcZ@7?G(,׽K'.xI }BݢÐ3zdU.iۚl*ump-UhmŹfh`iA kh`po-΍6h]k!7ًϼs(=[OE(*^?7wst<\RxNWss: kseLOԶkǎML~>帼D;#pm7fICYAZ @;|L{ht֎H* 7Yc=)&ͮ>Ns-ε׶ւA-nxRnG&pkfxs>JsO|m q<:G`뱭sGK,1Rt:x"{9Ж*U>?] <ӣ3Iψ΂\o>0 '䯣3yzl9;c?-O?(ikt $\0|k "Õ3: 3 9t;ۥ7{N#pFvj8q -mCʶ88F⼼+=}Z_ @%k%ME(,߻կ:c*6BIsЃ&?y8:6LGA͞95}.mIth1,;'˻;:oztA IcYp߻ wE99~/2;odyevQt@rF%!{'>۴iU6mZhv-]Gsk3[.~ʎSjүD(* G\#^L]wΑW^n?3驶S'FgA6]VyWpК[71~gfY^z{; ,1Jo(+yCi&::D^%YБ;ӣ?Q9]|〛:2:qIlKNBH VRm RFViسe [B٣!aQBvȠ@ a2H,ْҹ?ez$rsٚIF IDATq$\?))LDDTnt\zW \h:E&.m[/7É8|bM=?7t?,zj~kF*e:%'_.m7?օg%R[d: ? ̆t""""""""&N@|e[e5L.?hBQL02HdGanŹ 9*Z8o~8w}/'_j&""zj9vY7b04E$jٷOj-4 UYXo: e75&1#Q/53h 6K[_:t*A(= DY ]Ih66^\r?d̐wW+@]Lg!"3eԒ}^p[zБDl[n NZR9/[w[0M ,۞E4=?Lp*UhntⲬDMp]l! UKբzY" }35;Ck/U/ES.Pf:ݎ.0hܑLn~[_6W^ h_P ej xtfma?tJ]{BLg!"3e\eYУIc8=ZF Ź?f[7iRW0ҧVt"Lh/*S;ϡso: L>MsOeý(tҘ !"Rd:v~J|OC- ~RQ] 3/.ݶbkN5m_sWqF=Tb[A0 !N#f9M|-26ԛ~svZk!sM:s!N0^6"ZZ毩v0`FY$ZMg)D"u |ڻ=tʨr\z]n:n [uwe: bLӪ=5 H 4HD&z#GB0+(\gAD4W}7sWTU. V"{ݞK](3:Όyݵ7-P|*}5gB /$5zNv2CoKvz|(✬A~*G}rwF0"cƼv{ 7WZꇠ^ٜF?Te1M='QZ9zui/C{YFĚ@IGBK!+\jPUs9sDnyh DDEE&*AIWLv\hڦ#*w*w]k: Vr_]10Y(@e?+&綴,\m:P!CRKۛO/z#Le wчmE>nw~9Δ̻m^zH5\< lqƀZn`CN>*0DDD9+oPl:S%+uNtSb^ tTFc"s4A!wUp}FԼsLe sV %>]ZBv0,7AD42RS}{=S<Gz|v5U ~Z5h06vosPv><=a<`kǀ""""""""plC*[npa #{[ `KyҲd$cNhu-ν.ٞ1{,{}xڊ4嵱u7Ƽr]ۼz< A8wU,j=Fg""W[>ntօ iqloMp"}>H )dWӠ]S![>nMuEힾ o|WTp[b:Q6T-7sPZDEfCoo#LG"Ksz5 ә(5-- WCF}UWMWU>OKP `wy<9Sʖ]g8TŹq{r^[Dnw,;hp;.&RiCxWnqDDgrЙQ4d5N ךzN.^OstkM Fz <vWy/z=5T׎5,UV?=VշgTh>Z7([V(3\~>hkӁȮ%>@n0t&"""""""""J_r] b:U{j 0'_T'_?o,dJJ@/fƍAD}Ϋw} /'ru6?f[7^((Tkȡs8[C7#Zy=8tQﭘ`hQ ΖhxMg7T! 0#Rd<HD>RŇXGF.7/"bcԥT1쨶~"{m79ہHj!L .9ܱP'w*/rW0\?WUPuAۨ+z]3]m|'埽Q^Y^箹"62b8W/ՇbE77tl-[KI*CpDDDC[;=q@Ǧq8K*k{gy-f'9<͜o{DJ 7|QiǍ|&| -]rNfujcJbl:}U_l3 )̶Pc XUL؂s[ 2d {F`ɚ7k׾ޜ "S]Ϻab EFd+([{5cZ YRfYr?_""'[Z}22)NkT/`VSr}6rwc;\G%dJ?u!PF"{[u՞C1˅'sUUuPqBPl'Cԧvz @8w-b:JZZSs@4 nPhg;tي)b2<A60vl:AՆRn>neP ,sl Qf]KR9L F~o^ NРBuG}\Uyk[g:\<GԊZ1].'KDDDDDDDDD䫨טBH!C}#Wn[L2_6jd?P h@8L dɎ3dŹC%?S>LeKwWz5$jqN$ z;\`m˒FŞuD~`,oj(92joýL4zß>+^*Y 7AD T+,4D?6|* !&F)FcSS]pMD&VXa%ǚn98֦Р `_]M&l4pn|5B}@zݲ(7 iwUҶC.ZUgK/vyPD4PZ/]sUCsP[7:QPues8@ ϩo5kǍzgBDDQ H9HٟM'Hý^wttBQ6)x=C٦o#sOg\""B-2(z^(b-Fg#9mE1 `lKXŷ@$B;;!Tl:ZP-jCuWl,k1 F^}6Lg!|T:Gjpi1|zX@ح"XbK; ] R**nT@>@U-#'4#""89wybW]O C=fZs]|&՜}Q{U\DeŹb'O>ĭ_|ɤ!""0]KnQKVꜰDU* -_:Dpy/Rօy,DD ZL "E7ڇz,g @tÏ5tyvh@TgƪPeS`;?JD!M߬֍. m""zý,9v¡C)D8jϥp&2ilGr \ڊZŝB y|NK- Qx~ s8[0#5G-D.6#9Dj1rYL\h6(U͑y !2t"""l␳CCvy> NShm+h-S^JD4XTu̜{@-nwuAx^DDD5]; oLp0[!*O.ՋDLp$kǚQTgƴxBDdbZ5(BL """"""""""JƯM """9;P㎳ /?sZ-ΉhPlxwNmHue9ܮ"GkqToUW񵋈]T\rw \h:E>[iX s8TZ7! ]0ht",lL]^t"| {n#9R6RBl:QpPA![;ͼւ.d\*YhqZv8 eј}Pvs .ZO:CgDD4h|5b:5ŬM(H˟,3ázj&QBu|9IZZDDB6Q~EDDDDDDDDDDdTsd_xt"""lC,aaK5E0Ԟ{;Ș1s{RD7 :o4-=GwD8tꎉOh:ʖkL(˵1"3érHm.GEn2(+#ADoZ+DDDDDDDDDDDR].6/IDDD!Y48k6D_0v#hq8-8LGF;iFg-=#N86ts@xC0árצC8Asx_m: 5K!򕺊~ """"""""""5@3hqȹ@-n֘vw8$@bqMyӀ48̓D-:)g8{=!""JIE[Mp2[U;IRUU !k5ЩΌxt"A-b.ƈ DDյ"zDDDDDDDDDDDТK6h0qȹ,QOT EiTg8Q"CaiJҳ9MkOв9rb;z-L04o({]T5á C85 ZĖ#QYo+ DD)ADDDDDDDDDD`p:5t"""!d{kǧc',%u? IDAT˙g8QZ?wZIA1#>cq"""@f ~n:iL/3`] l: *Os8ASkO"*zzs~ DD֙ADDDDDDDDDDg`,9{d'_e/Q4ʑgQ8_ǿD`c3u}Ӭ+?d\u_㈈(%"S]mHCG7|b:G!62w5é,"es8HN3hc0DD$ .4(R*Xn:`XeKv|%;r^X-"p&*Pf?4ǣ۠32lm 3lwƥo|s iʽL{`K#e6 "tڮm(*]@1b.tݚJSZI11 rKK-$A'"]38q=!GQ[G,3ϼ,oC5DMᆷ}?!#r4w uQ]u'CM!"J h}M!"rprBDDDDDDDDDDuw+T7Y2ye+K%}[uK0nte\aMJ1Ĝ-ΐ v_:4BQ5>72 ^g–3WO!6J<?0b;!,fueQvis] ` N}Li- V@AFl4z@4e"""""""""""cTUQ"|j: @]_jw"@\Pd1#ŹvYmqN:LylT`]g*&7jϬ8tI*0+ZguYXuQ,qE?Tߞk{q_]C1Qߏ'm֎Gͯ V}?oڔC=^'ޯ}]퓮w[}I>gNwlf]ML6}( 6` 1C~V6z+,~Տ%KZK@'DDo$*pCA5Ey/,NFȤWꜰ,N27K5VS'M!"Tg**j ` w?3gp8CŶVnMY#l?s 8c73 -81OLZZ=s8^RU5qG).P)=ycDPQ|MIx!?"""""""""" Z>Lg!"""JWN 9mcU;2:r@mΑ|[31göO&#"ڻΎ"=f9wFsO~v 88 Yū.2#\ϝ)7$JɆReyhl4u7c +v9fŹf93d?~S{W!?g3eh__Mp{7M /m:c\\YYN j: /ڦy Y%fه@,DDDDDDDDDDD QL """J&k$GϿ@D^t]*QR:3 9Lg!"""gPm5opq"'tޓCCcd9-Q%>Qf̰]wPs]#@ rsc}':qStJH'jq-]6h-hU ?#B\ћ`ZcR s8MCWe:B)FBDOZ>L,k@m3׶Mu>ylm6Zs<[{<x#$ɥ*r`h!h#%hHEncP'պp\>"/BΌCDDiXdufYR\#pP59/O-@=as8C_޻>HcQF>ăݥ\Vs"lm6ZY(-6+hso[XvWIQȑܑˠg:iL/52Cuf ^d: H/4TkGCD9a,=$t""ʾȼ5:YR\#^1h9/=D>#YD{g[vgMi(m[as8w8k:SbBUysPw.!^ iL 43 -֚BDDDDDDDDDD FG(,DDDD@n(26tgmV[31{-_gR -<p[S8/[ɗ8ȡrKwί8!S(^tktUm ޓ=e^W]]3k 5<(bW=Yh@(P/CDDEui{0\w^^Qg[}Lg!"""אУJpM6Cci;И,{4~rJDY7૤-q"sx]#I r~ޯ6ySte 92񛼅ܥbWI( qs8![i:eOsaLp-\i:%CDiR#g0(DDߚ" 7Aզ9ٺw5ۦv!UC ^xWu7 YsmH<4Re،6G:7>[-=ӹ GLt,lzcO%mcVZquw*lqxmA&ֆst_z<&]n˭XquuVX}, *.۽Ot/6n[n~b~<>i<%2eފ4)>o7eDLp*Q=vO9(9պH喍ZCD)"pQaDf*P`,DO,9t """""""""62wyYtRi* DD*!%۽ [9vhTuIDDA-S-)4ǽ G½/2fl 6(e bc}ذO k Z^pz+1UumU}V]nΛEk IDAT*mگtunc͵*=o}WY\&RltMP :SVʠyC筼möCrP lbZIhs֕ Z9r)=*l>ؖ Ealv#쪢r_(gK~Pte矰]+%}zhgUw] `r8P~xKޢkFLd?;_/_u3Q`=֩g?dHE?VS 7$&g)u͏RhMO)IV]"F#qmXO: J(}z=9N)qSemT]Kj`d+R6fnȑ0=3l^?G/$wncYN mru׏E8ҙc6 c tBuLw]w5E1N< LVFԕƖf= +v/,sD(^#B#fv|G]ŔŹ'ߊU;fl/$puJת ;[fXt==8o6{(_,[m.0LTjNն\z*DR׿&- j}Q'4;z`b$6|۟j_|fR4^✺AgIn=ɴIqNVn-6?~0)QkČM.)3J+[嶡CLq^cL1mJQw ,E1{\+J[b%M 0KZ=%ELE]]m8'!y()[g?rJrmERcӭtvqƒm*fB?$:"0zJ*S9-RE75?o̾VK osa)x||E@?7#IkVsҜN30wژT[$Ջ~a^4?o^)Kbo|Hze38sp֮+f7 !9ilg3Y,'8ww>o|Q{)y99ks|\1C3^n%U5D9nC~^o a҆?bKW{/;L`B=:^truw4jϖZ(`S6w7|bpJ}g\|}jcy449#ݽRnQt=@[lUm̦n`M0b PScӓIq%9i.Mcq J{cG*)@R`A5FsG<Ź}R6I7.9:sr6Ƀr ~go..u'nْ:_W_+ OK䠹EpuR+O7EEƦ>Vt1dQi^kaْTt-@zp|mJeV @Uwsn/$,GG\{Ն2vI0"o~܎rg7gsfr8gXMcc&zg> {fbM)Rp3lܹFش]sRzrg^3G͓zrrmƮ#nGmzS 1BKA8~"0E1[.|nE?g t-`\Y_]y/~+\ݰeY7ܘg}. *?Qtܿ\t=@NwJv^^#R <]L&g%)!9)=9eN J1tq<Ԧ%3`lw lT ~raސ#FRn[K$%瘤uy"Kuʹb7rm\)f8 Q^:s:V+_t}'d_'0 >hf׃0wg6{^,E]EWTWt1C1{m3in_t-@W7f_o΅^&rv/Œ"Rt=@J[YzyW/mk\ 0MY~f4N}l_FigIƎFKa[(8޼n~8' ʹ!Gum'zsTcu;&q-Nq.<ش#9a9yePR~7('?8IuL;+ ,SVI͢bg7]R+__m̝J8tc19gaaF?/鮢bmg?ۏ K]cݽRnWk7-:^/$HiMRrXRzr4ݤ^w)=4(R$YqqO)1Tcʱ7rnRPRd$%[sRMZe+Z;׭$[QGk_۶<6(ҡMWfn6]GcVd[tS=wÊ.a6jt3]P}2g'ܝLj/Vr /ϗ2_p\~ۋhhϩ6_nl tt|{Iқ?qMc v)5e3LgRzm?7[coLpόސ#ylq8klӜz<rW ˍIq>0 '(mjnRt埩_(fpw$Uc4GErud6jS]I?,&`]['Ujsu}]Gs`K?i u1wS'_fg0\s۫gl`rfYU72hr>481&)fIq$ wwK͖e H9Gƶ1))ZY]uPs5)Mqnk&VuXznRs77<>fQ > iCuLjEp^:-/|^U`wgwaOuׯfgL}2{׆-KO;LWk %7YHS r0<>fFZJckPshA$0Mq^H`~L)q28^2=sRR4^<* (yF-IN e6ͥ+3,!:~%?/%E1W=8_2 /(0 ?hvyB0_T'LV5}dW9J}v.]bR=7^)?rŕUw3JAH0A(?8_2SQ@Z5]]m3+'C}値z-M;,wzL199gJʰ SG Hq6[np^?F?ESO(8g?$ͺI ߹8>Rۖ|tz5x[tSnƆiE`>l:Yo梋p:2_H1w)x .jÞ8_/_ktׯKzz0s_6Ι_ah%=4 Ǝ7f_nɜ1(GLd?1_/_^>8]C[ \Ks`e#1v)hPgIa pX 9٥&疉-Ź 8r8w+ )mk '9φ2ݧQcFyCmMoȰ yKzBuL37^% ěIZtd۶ ^t>w7|R=BHҧ%5 H+r]^mScTQ{Eׄ_8jw>Tw%m(RM;%}*,_;p }HidkLYsIJqQA;Mq.ب~ KqTG7&_Oq6͠R?ۆ26ʵa܁8>N9K"]ǔ@}NJ.!i:؉0@W곯ޠ'jYR=Ҙֽ|mϏilzUEׄ1Z{mR}~bQLr6{aE׃vS܁/ҦNJ69q;NwOH=ܶFαms +Fu<&i)25G)Ϋ%9x)}gZG~4ǯ✤)kc6K*e~6=ry%aHTtS,ta`-,l]ǔ{q[.>"P}?ڨ=A(&@҃rv|}Z `R_7LcS$E]MJr%yЀ]bҘ}a>+SϾRᖢٿKUI(MMO%ŹǦNqH%Ź `+MCOq6)Hq^?Go)vT\Y8nw=75O:1ߓȍ]a`oL?r▋_-)`ZZE`ɱE ./]FըgR=#p{GR0U\YwBq]~bRP^{jn^Q.iu+̂g1IKEׄԔ³˻.l%8Xmr6K=j)IM1cMqΖ:GD]ض8\kБy0-oNJq:OhS׿Vr]R=9|׉P&ش@l5g~Mն1[yE1,^E`2U77?,)}KE`owVoذeђYW AyX Qi^Rmu. iHVjcL&P<%jÞX(57w]ᛯp[1,ņ$ׂQ|Z>Uo-nkqH͏yϋ>-&{a\sfr8gl4zl vzi#q)L NN\;:*c#0R3oE)ssuW^))>_D&'ט{2:tnRMdk{DL[/'jtiږM-'cۻkEׁUY >(iwuL3t ]tM\gm3>I]&AI46|lQnvE0ݫ8_{S^I]7++*ٿr/,8?L;-rdWO^YYqOڴafe $)gKژA))0A)~c4')9ylqS `\ >Ź|tν!lȑ~nk&)v3ُ7I0R}Bi)-uҖA 65(.[a_^6_3]hUYqOQ~F &o,& \d Ϙ]>/>@l>[t ~ݡJcJ}Ѓ \[I@6jOVoDTt[u[ JtSaCh8gM;lC3IS@QLq{y{{:'YӬCsJsQLq^8#i&b>8N/AjZ/$m)ifosu:0*+)K'n¢oe6{eas]G" ]h,,=R={1]Hw5s3_ l6n%WHz1Mum ʪ~0:4nZ}G;_/d&r{U/4>1?覹kQ7#Ifs-UWKRsQIq;~4( `eIM.^Cj7ݐ#aQGMX87FoSǨZ\ySW<\Βw'{ѥ&{euL3ZBu`zxӮHƢkR}縗.czϛ]^:qsC r=Pɿ"׵%|Ɨ]݁;t%tqz~FqE׆p߇^Xbz[/a}~Ӣk%]{1/xטkdzzѵ!#&}Q >\i&g\o]X<;c)HqN0RT~ؽR37fiTMg3yGy%Ź)&X9wVLG6_""-ifK3*L'nE2͎۬%}:0?єs'{ɳBdAYB Ġ=(ӗ\vJTo"¥kmKuLWi~y6Ӳk$iI .)#V>Ocԏ߼:P0pK6_\t㖱Ӊ./w.Mfw]dSo)hX-ko,yw7a5 ^X o+lv0V6Z'nپ|'\5%%Ԥ,BO*l IDATnk]8s6[tYt d;s[pL*⛜n+=>061)99){"ƶk%4ݴ>>Tߓ7~]0B~W})^!ϛ$ u/dx[ƮorvI2[}垑"ŹiRpZ7Y F"*.Mk3vkuRAu9vAuD]GINkC[Zw`ϯ3kϵ[߫VzkIKU d"I/@8 BI_ t}I%X;{Kc/ M4VtMPew) 5'l9g\zg]Fi>Y]@3MG) έ8G"9bcK31= ,qgY4#9B_S*:t7?HqNS#COqYsSה6#Rǹc֍PJOT 8pC]Ҷm=#hڋB.B D&͹s8P~I?]^:煦t3o׿K9몵 ^VIJzꉡ,|.FE5l 7- h ,xrilPtvb5woI76zT$ 6g9{>.1uy#QsC{lM96= ;6y,c﻽5'^g\7iCعZϊLشRl(ڴ7G2|L-%z0w;%]#I§CIZ $鈤ig>Y޴xpDSҎ>=#/ۥ^dcp4$mwϙ/]WYqO_%}@Nx±;CXҥZNy.ZpIߔ:Aѽi5cp<ɐ8#gȨ)ΦfOaSs3;mӏ.(f8gѥ9>tYߤ<%-lΒKMIwۭR57Qm<궕lrM+_2;mqOzYp~,Iϔ4S\#c[%e76vs '`VayKdv̱džO-547=#p=MH:E#uul+pzp`V>%IںGo0;B;Nt!Ӷ*\M[][7|~bVmKx;o-O 䧻$;U +t~@vxpC. 0f$Hl8'7( *9猍BjX?@Q. #ܚ7]dsʱms79d8cRg_AgsѶR3^-û~;|e2v>#E";WZ<l[ZdS'B7c1.{GK4䐤=ncǦ>Y} t|Mw]$@/?cIz136_axdGju&>Isӷ.]Pc>iQVV|k=E"S~LO/􀛾g. ܵv*@qk!CMqGRद4i,ilbRtuHIq iLt=3>9/R߮K"9ҍ?Ǘnq))m7pFc˗r#v۪$}v.Hm嫫my=̛vL$7,%?ɤ͒N4iKH:Q˿nZNJZԐ{=hICrg%Ȥޅ6X#[m촍'n>QK (xKIdv$dI۴I[Ǫ+_*HJ/>iឍ𡇾r :1÷|uض‡op=3|PI.?YmﶷI:V}\rJȗn[REfU*p_(HQsOwoO `(f\:>]"pʆ)9{hPOXRsoҏ)9)&HS٨} ;7j 96#c~Onȑ3Bn?ó4FM!87;^P'If0Vs7 ]r/4;ۡǹfŬZ`cp}XVR33; NXivzܜRɗ$>oaa>I$ݞeܣ%[xtxfkӗ6JҌmiDJa=ɞB^nz_2M4ޙf BMcM00.ccbsXRL?.đey-fK~04̾)ΑqqmC IqΑZIOM$i,Z`Lq1y:RS3Bi&N7~||nXR޼Ad4Jq84[uц41i @!A8Ž30Rp{NqN"=zގg'=4ZgkhFߘ;O}~Lu =XԚ%9_ՠPNWē`OqNސ#źHiCuR8gg*ŹRklD ~S8ǒ PfR ~h8v8gIn3$0tLqz.I 9zy/#%Jh&nZdgo'ź9R)9b_Ik߆25ŹV{/JgsQIq;~4(P$5T'@!ҧ w7Ҩnf"7.X'>=3qHq^Շ VBgDr))&Ih8J?,iw&o̷Cr۽N8w; Hqng F ITg'5(% %9bng)Βn`4un|v5ĤݘingSsͼn#✸8O3@M7eIqDsǶ9)rَ @1%9BBuly6+uCs|_S#rC|.)*$wۙx0RS'A=UF?S "9Z׊l9sIq${ HA8iP*4Ź Jc,i9d0|)S-q{f-s'8yù>tgֿyV)ΫKsĹ,gaiJqN;}MqN^MR:&9q$L3@V/>EsHqӠ,}`LisDs0R4^+޹)Ω✾!8~tc;\\@ 9jy̗%8猍BIMchPJLf#hIqn#!8{Buq8Lq8QsF,1c[~X)4<e̴;)Z˝<䦱=4($|~o0ȦiKqbX)q"Ʀ]'Δ߿Io v=~C&}Ox ZB}<&cPȡ ŏ^X4FIƌuǴ+@q-3}dNq 9R8~ȲIʀ}l|5#)ν7^Z/{qMk%i,}@Sc8o⎴˲!Gf6ljnO66úx=\)d85,F?Ź}?fIqΟvXPski8N],)U;_uY7UuG59b8wlgM3{_x1)ksIsĹE8K\<iWѓ>78[嗈uzL&=3؄9HqшR&NG$=zHs,_;8=8'FsIIqx= )Ή19rh?R#9_:kseÑ7]G$6VsOij6E`LHsK]qO/9[v,rRb,)Pumr~w^vȤ,k$j&2ŹkO* F )Τ8Z7Iɝ\LskR0 ]%pIZ^b%9A`Js cێ.Q5os:ՏMR" 5Źd>ؑOqE69_Z៷>JscliwTIzo쾹CJ86*)ǣ-8i88X/)Y6HZ'{srhMΒ}@ҏ$ܥA)ZRX\Tf Jg)X{)]ƒnlHqN>w8oIo,RF]l_Z7,bsl J4eI7iKlX~gϙxݭSP'&)qzX'1ŹF^RD4Z=eݶė+k 0b%^[9s8gl~iu$Ai,blt$iM Hc޴Rӥ'8]uݞoq_ScIgq,%5۟XLM.R8gwj0:VX✹r)Ή;﻽7ZM_esSq)YVs=MXg,߻>|4~4( 1ŹâRM3n#&*9KpsEkCĹ7'ic-3ϕɝ9G乣FoҏM.)(B&gIZܺw%{)c*/aMqnYN8uCmu!GuS %9aNX@7ID3H|kfzfc4vS~W|7ä07H7Fnȑbl0S;J|G+9OORhC\!`jnr&{5✩z0)Ο?7HZa%8əjCu)9kuu')H3j%zm!:0R37wJ3,In=)R#$893:9QŀR#LebRxϔIJޤ㴡Ls%`:dnr>ueL3v*SNS"o}s`\Bw+a8[{]iJqnJsTݮeP&48`279K9w_zgS#Ź Jclqۛvd8X8BsF8L\MΒܻ_vH8G;)ή~H?(8w+a~8'L'Lqx)9)ι>tc)ogE&gI:{/ƥk!ŹCRXRRخ)?Wb& ['rlǹ8~֍um2)lXs S${SLmso vٚƾ8Sw_v80Hq^Cs8Gѭ7K S㚋{L}v+ )S&涭H.9&8CRM7MLN IDATi}d( &@M&zh%Ź˺HqNȚ9I/))]i.y5ԟ7i,)s$åC~zL;`ܥ^xoz 0"Rpe/)ι7蜣)ΞuÑ}֝{mR=:rR0,IgyydyRצql+6yotљ/{2@RۛvQ^{R;I㞙:9Nu>yTRȓ0Źk59KˍnBO=ǦƟLq摥3n{Ӯcm`)YS*i7blZxQ5e nk36nsG+9պCK!Os:2ʐ 0,I\DD"e^.톑Ѡi89,\pַq`H5Nf/ސ#ylql) Rx5miNq^9Hq$}or9"o4.)3]{dcg}*iW;}vJSSh&̱ +1H8@jr>ns2 Ź/)Kng~uő ~LsRG%cjjR,cly6(h$=:t;['Lq)pS)Kϼ #u0Hq8v);ǒLss{0N,I˿i1JG%Ź9{@n6tu4(>jOyR0ne@86}h_R=hTS#FY$}Iς1jN}aI~rCswez4Jjɜ91i ϸ퍻*igWy{|Ԥ8'yR։w39ش ˺Y=řgIPhz~=g_^KגHg/Vg)IcsoSE]ې#C3q;k=9zhIƦ+_CxuRhTG;玧2Iqt'9{Wқo K,_tMqΓvca?./o: 7ŹDՔ5;N״[Ro|sDm sts\Hvo(8`R\QϻH[]ơrJJNqݠ2Ymsm7u3ni*$x?Ncllsأ"U['{sd }gf88[ {.[-c$hb"4T*TrhD$c?P:`R.- cб)H+SBRf޹i""F'N" t>4'xe>>Ys8צ:SkF`oqr-xsZQv[/;̭h}'w7L ;sKݎ]7s4'On^^L'S͍g<;=N2-ΥPع#Fg\;C38ȐsW}j Ź^9yym8+洂hqu]suyl:Źjm8w]kͮSN8$'>x[Ƿ8W(8sŹ4'<|ڵgѳjNiP31r`Anq0w-Η&8ͬŹ;bS-έkk™2~ثW3XRfZ{27\ ZOo5>neq"\o-\r`1&8nyp:<yֵ8W<`Lc٦܅s/gAsi\r`8O5<2lz,-Ct=CZsKڹZ3B,B>:}6\3/zs{TV8gls~ lqnߺ=_-l3 Tmoqs(-΃ӦmqyB8~}^=80/!gk-εh_o;X:Gs}mq=GuX{A-}{gU?wR3 S}iq^Ź{Fͽmq\nQ3sr`Źv1x_ZfԬ\-Η8#B,fZ{<8?~F-8O8s807!gGDa8] Yݽ8O=GC8F-Νk{/goq|o9u[1Osvm9ŹǾ8Qozsvmkq-λ-|B,H;<׳8}-=Z[36<﷩Y;Źz81i E&΋k5Zǵ8^kqn}97hnozx}XWUARs1yD-=AZkft޻Wh"69opNHhqn"rЪDZsݒg|w!Ϫondž?{79焜XTm^8lT}8 -kqjq^ Z]A49l3n;Wޘ#BZ'8wsksZKs k+殫oqXst-)g@3 Y}7#;8_RosMsZGY8O _Ψ:X纵34" kq$ndž?79sB,[?tx8"&8gZjy0q{ ےG]kqv=v|4r`YR]6p{Rƫ 9,=t'EmRsimqD򚺵y-Υp|ܵ@ }=N%Ϭg1mc焜X&5?}e˗!kƇk8g_]sf-]Nǧ}0x6Aor`q>5"\yIgֶ]YsޢciqNqOocЇm 3ƇiR38_>뚵=\ b*LsOs?gbg"cI!gW@8_3j@Z{lŹ<\7.E1+B,g-Eg&æA0qHR߽k'8轷ͬhrZ-3[i܈ts" |ZqVLD&N[3xs#iqn ZkqMq36JŻ DJwfjq΄L8&nǵJhqvcpIam8W>k/\N4DDmwlWS uBwo#fmq.S4q':Źi9&yS-+Ź2hu%߬]/M;禹7{Ƿ=UB?zizoH#9Z ^1Ź3h}\-cZ4$݌|'tgNqߛ[iqnvMvݻpX-Γjq.Hq<M<t`!gGJ)Κ"7[&N;+{lŹfǞf8kqIqyKrvZMFBݴ867|:<-ztz-kK9Is3;noHш6!ggԻ?XKnŹctM'v4'wLiε8WW8ׇt8׭U79g5ҝ8I#ҳq܉&E*yLzS-i-=?|vT܈/oŞD|@&p鷽ONүG=|fluf6q)"$R>SD#r^wg_AɅb5 /=I,)SwrY}kg\t\qV5AM:i4q?/‡4^KO+W5q#}+֚[Vwki|9pٻ5'EE<|q5@ʙ2Ak7Ƶs>?5=ʌzDu᷑ϯXWy3Swq>7ūӇv}8*O|~u/E"y%{;Cιs9׆6^gн!癞յs‡>+FTD|C<A(;`㬉OSVvKJ-֮٣<`Z87:ޏ\՞Sjq 0wz♙OopX!g?_jW ͼRsn)ʵPj./hqΩ ^w~s{g&8oפw r(tvޛ*[y 8jsY8)5)>Mw}rhF:Źh-΃ֹak׮e[C|M|u&ήpBkNڈ=9=G?wMlV~s<[95`zt`!g޽ox'߭o"~8"93RsZvWxzw}9-Yh&^ _΅87gN9YҬͅOE+_w} <'_M<~巳ݖ\r-vKJ-Ι2Z,Wx3rsｷ_`D\ Ȯ-ͬ]o _;8x W _&lB[<%<}nEk0̧II<ُEFD.N+V2@m9r.8gڥ3g,K-ιs+LnūӇv}CFz^qMčw8kq\وOq.^>]B0ѭMM""ueZ kqWX0손3^y[R|Wehq^ -ΝCoqZqE?/[>% w/hVJhqks-΅syHgmujwrToצaBa _xyyD<">"ݻrNZ+sԥ\:|yڥg9O"/G"mv} 3lӫ9OYsڳ͍xIhs~B{5oq|hDĻ#ůǍx{+ٮr]_rS|qs/h"ҟ/H'&iDgDԄi8_yq;"nEcz4">M|$x<>OwxNG;@?zIENDB`xgrammar-0.2.3/assets/logo.svg000066400000000000000000000155701521764210300163260ustar00rootroot00000000000000 xgrammar-0.2.3/assets/logo_square.png000066400000000000000000004635661521764210300177070ustar00rootroot00000000000000PNG  IHDRN;sRGB IDATx^ -]޹< ,;&,˦! ML&tւ4INN@ D~Mw @Km -[F6$y$P e?!DzL.>ϟ̲i_?(S̫U1~繬^xs7e4A|=6 t x#1GBhbRh/0t۞CZW̧0W@! @(0 |og<36O4q؄ !/,kAmpJJ,`;mxq7u!wpV츸k]ʊ|:4'g]Ǿ?|_P`˨eP @T 鱹&tǟ!|{z>N+L \풍pу 7NUpGЏ K:ϱ 5!YOgGWŤVJJL @(XT^?i~e&<-wmɅ|tr;8vn}%CaKp}k ,xcoP y5oLe]wpI&v)^#` y_3s?мɿ0]W@c(P`Hۿu:;ih? !#? Ke+s5²pAsm9v M|ƅ7K]@(uϾ ~94?M,8U=sSxPTBʸ4[,_Va߹*Tϵ.6ϭ'B{^s.m( {P`3jBdw<]*cpwpέN텾Pօyax*}ݛ_/Ln! ɄP @*po^bC{; \5Էk`pډkgZ !:~PoY@:3P @(7~11puQpR.X||.,.x%t0PY 0T (BE?6,~Lnݵ p|zӛﾼƮB(8 yѫ/J.0Xqa~<'.:EI;hYyt~x^˴0s7?6Y 9;%P G_xb {m?=Ӿ:I˺jx vAd!ǫ juҋqg3 @N*HoJz=78 SW]u_{O;f1P @VS YGtaXxGuBpPmڛ/&1kBhxc 1;;j?k-\휲+.2n'ӎmtrify-sLI&)Z4Tn;+e3\^>ϟkn׳sif,nMfFm/Tv_~mɟJBFg6B<}&ĵa ach ׽ ZrJk/aʬicO]{W]uf0BP @()_gG~υ8!ꆷN8x@D`([Bhbx÷h׷C eY_4i%0>0`1XXB%i>- ps0'`D3%R3Жޛ}?ZN<3_5~MiYv5W41ಮDeAz8-pU@Nȱce8L XN#GX,/Aa; "GR:A(!kbu8ZK&ւ#yu3E f&Iek*k9 -`2´=vy\@]_6ei~ <Oقg6~,}aOw.[|4CxYy_r?|P G^1] $UF<#vYz-U0}hv\EC!0i)jZ 2DB`bі< j $x y~?Kn?d' )ٽqrB2P.9I(l6>T^mt6Ӻi!v 5\sU= P!ЭOlHjS(!fYvcPFV`dca)t2OZ V?M'@ahMظp[[cOme|+)jBx"67f[z 6 w]b P`+ЖJ9r.6߄Ct! rlh(Ai\?-H>|"b5>*s'aZ ռCFjJ%6 e'W+snrx+>'1Cw \)ܓӅj@ \ eP@ i- -em,7# e2iRƣ-|GƃЗ͹0A6g/×owqP @(&Seb3"VL4mSeT(K?qNuT|B@IIɮ8xȡjHj:پVQ.y}~27O hK#q PX -p)\&A_se S'28\; sU(l Hc'-'wǷ굞~ӳ{vӝ}9a.$P @(j |Ӈf !;{@C;1d2P\8qrrʴ53a;6H6b*Bǭ(  5 g%QrޞBl߄nNZ:6g )P j&T 8~Xw L!ou̧>=a'C@Ѧ\z>7&kӄSyOYw߆+}P )T7|m9YqAWbo ۿMN,?)L9i'ƪ@ rHMBPA C/awYof-9w* YP ,ݗ]ڿ qK!n.\a4c`r+6*0GN! n<ϐ1pLlqd<.pU %.Ϳۣe=h>Bmdj)  @fe#{ގ1~S薩Aazǎ!O y\Py9u)Aa؜H8vWSÝK^vyMߋUWY  ]@(_ ~&V܂",zVB%{4?.;؅E;F-93uJ@4fIΣK_S,!em 9S\E;uLN@aa/Or5_|h}#c=P @(pNa,>#fle\ %6$A ^ź }N!o)F4aG±?.%By}l:S~x9ٺ'6K *  @(claG#*+ E(0 S)Q^"C\.KȕE C`:F|rث13K4M{J %eO59&Js5n`/Ej,*صBS,Q|yv"`nNOe4P(I*)sb俛o.l|t RuKnRnM8|uV]s,0(+go_xoC|Awm0e*E% 0hѺv&ܳ0?3~pf.kvo` ȹ)ǑN-Bd)TGL E'0c AU(j2i"Fm ]yC|葢Ҏiכ~W|WX`&P @(VZ#YsE '?OxJc?|(|OA%E*Dd 94}h 䡎3kMdP %Oh^581J>} h]DxPX:=B5B dp9GNq 5Ծ_A6:%nl7]sS?q|tg|Ǧzowl*P @(p V!o;hϪR[hA{&hgayrl \<+M,AP񫃨ʀFpi71*k߼ӠlǺm912F*:F*᧽cl܉ i \݄B~O vc8~BX؉zMQӄ7qv>º;pĖP [n[ڸ"+оCw=U͐vJu;(0g](4H;CP.qr(jy=5bXpB60j} UO IDAT0#1Ѩ ԌB!0QSH*0 s4jˊp kqw/xkmcU '> @(^P1&wX 1k.`cp#7B} %P+1cwL e'NCS}z#PSׄw4^ކVd 6rk H-8E5("A.\Y@:vg(!-P98ZL?q $/aĭBGLP/ʣ[Mc?y^s#P @(Mny-}C_&.ԇ^[gb $.iߛ"95z`+)BǹОFu .+PH-&ءӴ5q =YdǍ':ly e SD#k%QJi !c()nbY943i1 e_Cv[ V &TV#G<pm7ىv'0 ggYBͯ) B(8 ~ٛ/kfmϞ wZaS~u*ʹ`?&$pRR@(JPhGX Qa>ʡm.)! y\>Bao9 y"0`g2KY8A=o5e(ږ>,јSS 9(p8vuoB{IoC7Z5q×*#pP`L*& ejΒ \Cp9}Ųf֕S+Z@iḘSdPsP)]PPRa^GemYd~RH㝴y(AOD_ZXk3W0RE;1T 6SNpxrHu 䊡y( WٝWWuI鷢eP%*o_ymŲ}*G쥱6O噖u(T͸@L&FG'2pP IVksJ(^(,A7b4f34eP+;Sr˘ʡΑ+v_#zm[n)S: Z8~F[uQr U3 C(Xi^ܰ\BUtK vވ u(Ͱ4v]SeQ6j@nE p)j={V/`+Tm^!QHPJ |49EBւk(.") t?d(dw(PQ=,T1Z<E'>]u]8sϧ3: qooϾGWpO@0v @(vo|l[ *l <v`DsJs\\0T !YKB-4وG@ xA\C@[S87 RPu\J 9ФPP\<&hB}&B {RZjBÎhK)SƵ-!ɍ4+hݷ;L ?"1Z?,A**UNm)7?)M.;r|.v憳Ň/n{ nOP @(wS?p}Lw @ n˔]w0K;>.4#}c@S)M%l4rqY`=4FBQf(@h@ejۨEibP(A.;" j %4 f1:jQx< r .ZsPxOcW_|,¡p:..}>~F_+s?@P @pyk/ob8䚰Ct(\twAaq ik)/0;q&*](kA笀"?^_ZEt@h!:zmn 0b2DM q,Յ 55+X_+Xɡ$sP3ЧE^zrpO=qm{P @sf֯1<)!j=plte~6n4gNHV%MaV>8Au&Pu=0B!`nqZ1-j'*m7fJ=TWNjjr|MvؑTS-Sn@uuQb?m'e]bQw]sCX#@wPKoo>9쫐G8 Tc\(+8kӪ p%2 B3 hI{e(fҧ0y]MK {q Yz0ZUD),ӗ8`e.ʮ#CqhDBavNفB )CJ] +(Erf)D}zͯ?_s}8yˇoJ+^F* 7mmgV+qN@o-v @(tMhg % =Y8y 4|T \; M 9A˳ۘ>q0Txlޞsxnv9֝$\&KVD1 Vrٕ\|+.u M)9>r]h y|[Tj!FmQ %Kn1eS{>]{C;nN p1_r57o koUVzP @(w^z6u):pNܙʢU׮irgܷ<^)We9mVڎ07'c=)-4P!0PHJp1'aQ`h cP1&`Td8HtM^(lE-i$NCE Q% Bװ^BPs Oz= C@8~&|߲ϿRWv+{h01()Ж997eMy@A•@Xwj +M⨵}4`\a$JNYG@BzP0Oia}سEaL2@ DvWBK*(9Z*@j[J,?[t 9g6bJQ6B{(U>oSׯ/ap+@ȮrU@jOEc84C - ͔Ph\D!)6)JRM@@_˗樽kc{ WKAW vZQX(4i"(4! :6QڛuLh ]S۷KP8w194Cmgw M _k-gαs,8P @(yyW8 K;wp$d5?*sӏp41 U d_0s${&V OBu#A)%faulՖ}4„rh p>!$Ċ@mhiO =ՊY"L5/ZTXg 엃BS*+mQ%3᠒oXWs p7QQ=ds^k?W0*Q@) m'˴(Bn~kS@A6MϹ{w XRᒂaq(A.o>u{erl} Xty}YoHh!^:ܱC+!"@9'!K=ix2?CO#z eǑݻcȄi?[ >*Gctr vxoq $ S>^ eЦҿVM-.rhs( ;8|^B p]#h!0ofh=1@8 @(*+p _ټ0~dw"t8EC`t `|Ҩ3԰Q =톒rx'+0BO,CETuhANv9oMPfp&(ߋunzd_P @(헽f! 7drĺ>ЀWZiyۂ+! ~yxPM5iw"fgB4;rLh9|ji;lYW$+5cڦu3o03a)/PGM=Z[AukPRi%n <24/㹘PTn6/aֱhlJ'c@<޹8[w%^> Ͻ @(+p!~NV2Dmv=wpIơyf(tn:~" $PlB8HBy]q!=j^M^ ;4zw:|P@jK(,:4 Rk1RQj[X)_6T!j6жHPgB.l㠐Qؐ+js F8y-{?8 me?4!MlCFY/}8A><%(U ^ؤ@ we|z>gǺ|LuQ9س86N1BCmK =j*_ ) Ipisxhb3vvкޥ,O*nvv}BSv. );nYm%X<$<@a 17NO"5ѳ;|;n{  VP @(áu*%Ur!KW끶4 \cZhΆBk1Ag} ϶h Sz,\B #PQ=~ }AYA(:GO 8,)064[}aPX86sBvr@0ep؏GN{GN?FP @( !AJ 8u8yn忻cn T@~z,ǹ),\B {i+gZGЁ`WA'U4./m2|{6׷`'msVucR3hLEj^ΪP ռ1(+9UD'H;Sb)}mR 9 8m]AnpM덣8SpxwG{ V T~ P`5(p NSc9K1#PAJU7ͻӣM @P'bXcRv.rx :KB0FyT yCqH×A-^ǽ9mwHp[V/Ξ`ZX~Τ0ҢӴX U"2E &z{8?@8v^g@8}@8 @(J*0Jۉq \-Xp I- v;@YN&mx'qhUx,PX,NnZr֬I. Qhҧt(.\r,a 1u+ Kszrћ*R8 ѶpO(p m{V WP7Gn@_][y  P @(VU#/xb$ A1|8d-pYh1zg8g-3o  P @(VUZ=9dt̵s%!n!~lY& IDATл9;<3A LLD@'nZKc-a1= Mب~8<(C+{݂2<¡ΡžP w-l),ZQ,C@Vʤ5(Pykyg8d,Aѳ*D7DcX(@?m%AОA V(d' 䉑3rh~e=O.C3Jи$DvMSIB:*&l{zBB[h캂W@Z*b~Z-t }T(@xC%wМK|-2© ©ǸP @(VXFN!A͛ڢVb ֖c`=]GBYNA-4#븦Z&Mp:[.(a4wtQ$9f|+N 7|=oH)(j~.U(l' Dl!>TZXfT(c0!L0<>ZA!x/pQAP\`mqMA,`; kOG C(XI< wP݃} Ge-5ھqT 蓶 h<4cŠȡv r1tPXcn^oES F+0YqXk'rD@v"4\AWAJnVhBWwn pB7cl(+@Es"[ؙm"*`uNאsڲE!1N0 f,j)[̙UpY5Eq ]їPImP{)^ G^+s)ݼ4G9s誑S{r0)w ne_M=۪E^{͊~%iA(ؚ a`0Nf0 X*HvGig~<=@06pk_^ @m (O:n,v0nmI!Nʓd+\ZNyJ3LJ-N R,m( e7N!:mPR;XW>; xLJ8) iNQ CkQV4u Hۢ2G?>rGkm" ]9@8m@8 @(* xa uJ?-7[t%΅;螲V=-y_j &(&e|lRz@3C9&L(4_rk3a ͐EIsJk} 6nac?awVpa:ҵ f P| P`Ua p Ⱥ,RZ ls~t?|.?6ENߠ;}'UyK@V+MpQO>**ZEP?B['}7i0 ۗ>F>TGOA)2\@5VQBa^._?xB;wct(+@!4, ȅ1 ;H4'9բSkmCv -HQe\cbm": m[Πu 9OCR D2TJ*Pz\zZa/-* H~)MggdXHv3AE? C(XI<䃔@j6t \|!a`PV7%5&SмfB2r3d4_^,Jn ì8&DTCLMJk(oIP'K *|ZWPȍkPȐfBN;ᤚ+2-eP;Ρq584\%s۶-a W^;~D`@8}@8P @(VP#/_NT*.wz 9wzh7ph@JgYT>0m&HI y| M =|CrjDY'UC2;hBS47da;bdn`w}8i X\amۉO}.>z!#oQch(0yG2C8rY:bYY׵``#[/s X2wG'Էj -@6 1&#4ʜA// 7ybZMHކV7oP3M6\U4󠙐@O ή)slN1Jv9@^B)CYt [0;m'Ur*p0(+awp*&J8a_Ch?HG}u}Jؤͣ*I-[H/m#Pi+s`ᣒs&h@vyr2:w@̫W@NB(ĢȌ;4u:6T=q{ЄEf:)@sӟ D j]up{pB14P @UU@0<p*PbhE9r;[ȹBT?1ߚ;X5Rv/$S~^ŀPW~8( 0;e3v99>Wͬ `=Ss 94IWw9 *J˲S 44qk.fҨ3(!a@[a3w} (o:EE9 f P| P`U` lVs]u FܮAY0.D>(ljU劺DLPwؑZgL";krR&D)jhBr[^3qmC2o0c &.@u%t .-3.|60a^AR#Ueb$7Lۂ0ފ؁\&Aju910~/ O3$]ȥoggtkqͪ~'yžA(آOn!?p0A_xGW'D )RкJ),A&s0e#ϺaBi*>i; `j8jޡ-&#Q-Ҷ5E`ڂ2JNo(N"CaOhA̽_ g> \"'U~nzGVP @0ց aV9IAPP ռ3v}y M ^,[O.p^iz:4Pc&sȀdHePSmdȔS(6dK,b2AzO8҃ ymODI 3. ppR18P @TjuH W9VkElQ,;QY υ&zPQѡ:6j#a^Omh)W&x3 xir ɺuT>+>(ג[h 9u0Cd~(}%ԓǡ9Fi )oKk2|ڰ 3H|K'> -5 Єg! aP @wAȊVp(] $vJp,z,3"|:~vjAӆ`6SMS54;n@q}c{[F.CӒbO~6uC?0''C(XMD$M wPAsLaݥǼ5iBEdh9B{03P+2WꢔCpgP =iYM-8cᎮ?r+x)mh';EPr;mEQ E50su>DԾ_rX !w0k p[pB14P @UU 򳴭$haQv%kTb6;X;ثބC0Xx p;h=X~ټ"ֆQ/{! pLoAΔhIB/9A)#Om/ B3)TjB[ӰP[7yΚ™O}.a; 6 `P @(VOEPfӿ|i`Z"nW cZG*^ A%AGߖB&v BCSsaQ j1T-(c^x_bKIBZA*ꤺ2зUImg }6P.™O3 q밖 p;p*1.P @V@[rc i^]#P!zχMe[ZC UGVAGZn?̼JH&o$(n679 v햊 u3Ld%FA+ 2 ќt)캃N;vQ.}uNS)N!2 grj zr(1<օ#EiLA W.Lk ("lԅr~/2cI? ܚ;H ppB14P @UUpb_*ϊv[Yv-Sh59g6Q Lr1 5Rnp6p wp/BF''C(XM2)縃@Lܮ0 U4OJUptہ>Bet / IZ-[X*gj3}:rH̆:)è|&~JJ2.bB> !&Iu}Ӷl(;.|.Kچrf6 9p(r#C(XY@@{B.\b9 C08 'pw4X 6KaEHi#w0>%Lm5Anz -艻(=„2fY &0)AL!l_"a} }5Awch(eMz\:K Eգ IDATn~0+? 鑵=*}OeLfRO҆1hz2Բ (f`^MgBVNB a )բLf6UP\Kb+-~eh2:@8 @( +%0Xq*VXՄs`BUPH: q7K ]- pO]G+ BKD g8xʦDYLFx61Pկ/ ܩ@8@8 @(j*8YuU8eV$w+k]6&h;/AۣCy91K=nD! 8װGJNKd!Mƴg#?(nNa4^}76BUJ CI 1 >_Q}u_O±>IoIP @(VSvbk{!w'L;X_wqwpPmO1x6pE%g]b^#$EYx=㇝g= uP 3,-kj@?3 }4/{9:T@ˤ&/a~-ph3j(σ;B x3ylEpS6MH@7Hos ;HgC''B(Xe,*t6܂ʢ:!jZԕ(_v*ru] K!1(͔Ga< vw0$ЉB[Aŀ %anoVtpOBu 2,"`_iO*,esV )"Ca_}066?l'@8@8 @(j*02yd.UX\jVvIKɟ 2!rW{9} S`ryd8Uc`S(b2)^J(*HB$+ЬkX(| _?Me<+{Nx+N(>P @( 06!ѡ~~=v>@Ș%3> %NС-.P @( W,!M j0Il,ۙHuM՛PZ~ƃroM; 6 `P @(VOp*xxg W=ʘ߳aW@aqqFd`c˶ RB=MaZܿN]BZ?ZBF@Lj{zPY~C(N_',Ԁ&'[Ba¯<`p<"w(IoIP @(VSv w٢gL;z d؀-W:&pQ1q$M5Sm$\1d͠>AS΄:7B9|T@:57 #m_o NЇpp"1,P @UV}bγE^)'hs:`+s^ A[(ټTʣB3gRQu($Uɸ]PZu!<8wժ,4#KW4UF].B90XcW b:GvMjiv]X%Q*zg99ܗwEm)?hjN76ivzZ M<|C8@8 @(+@XX;8ja v08 8բ#r@se5BB6\r̅\wpZXSkf!P2ҚupQcja("ٜ i>%XrigL;I@f,O[4}n>X|5NzKN*?P @( ,v-8pϠQxvـ օ:qc(!J.ԓ*ag).)P4JŲz)z( !yEQTd04އh ;9zV,8 aP U ; hwПC;pJW(N Md)4 tql)0!P z159qZr 6Ӟu drⷸ}NtPȮJ(gLxT7 z8^&ܹr6hC8ٝ@8 @(*pb]Lk_#PZdEWGOaQGUp"׀1c!ݶH'G.$Ys`I.^8p-=LmroC/dm;X 82ye'7cp(05>t= UPQ$ NerK1a@JY̶K&.[ށ*VM):Lz}Di âPIB1J-*aGBp@Mw0 o P @(VUc98%E*r0"wL< Cz\zAZZ-4c\b>P yg3a-C=߲N V5-DnwP@g /lBG…CG''C(XMpEGzx@zrzۇ,Uu!ݓc\.7]tҧEeCE9?R$hwa |Aw8鵜b<..@h%okM5. д O<<`zd.&} eJ$nTku sP'SԁqMzn+As1<\YTJPo\T\*w8Nz+N*?P @( ԁ`h!wP5 ?[O{ϑxVZE[i,L: aBq^aBP_(&77'!oEaD5'Rq \ p[pB14P @UU asEhrQqVZMwB 9w~)qZ8K!$6+^8fyK4;fe˵7+f#жgmz[͍kyZtıj(ms;`ZNx3N(>P @( $ l+Bd `O Ѿ8V* if5e%QSڔѯ7* .s1Nz+N*?P @( X {r')>p+,䪙9@ QN6A{@gN 0.K_6dxTLV()pZPOCPS+ J$S7V/%(;揷a…C96pK;pgtVP +D}V.@;(ƆP @( d W41Hcz%m3Nl+#N麴0Lyxʐ䎶({ ϋ6z2襙tBZ\gi!6O37¬o̎o8v<ӏlh6XY<.GOp p:WNRr{p m& {@˅* =u;d,YR nb0t2!m5OCD q`4f~oFۉ DCS^c&|% v!A$:|mQ<C ͇aϟѯ?{]S @(j , V;{ߘEa)wp</P*4&5<׾'"iǎ @8 ` o KC'/i6=ߍ1³7S7F\+;迌Vp<ݷ{_^sP  yCB+gn CAws!7W0>Y]߲z-voaYu{\;CZ?0Aw#1wEt2{ mȞ1_},sn `QBBc&/KOc {VԳEv\ lPt-SCu,96up?{(L8 a~T4p'_8SSc_:|? *aKa_򗺎ao#!ƛg{~o;rcJP @U a E 3ruƑ-R`8]{rlje 8p]kDcw<iXfMױu/1is|9*caQͅ\O}m;aᥒhCM)R SGZE )! JCN:ЄcxI“w/9 #V0iiB| ׾E_[%1(*. @gwPb?^E Zȉ<1Z!A{~Ō,'K 62n[YZXRhg(q)Nf_ ɟkcC$jG l_Ox]zȵ > L<BM1E @T =A{> ;^#OWyah3QPB3|U Io&@A!fk!7$LP4ietSka~)*XfqӀ o şbB<6~$/^ڄpA5)xE#O9ZNq6bL(VY+_cC?]_u] np;؟؂^c?¦g>?ԗz)-cx=yV څs`;@/ZRfɝZEc IDATULS˘b1| $ UHmBmMat@u(:xiBsC8-@8~.8ݛͿ!7nr#1UozC 7[5JOvwsZ ! O\QT%=%FP`5P 3tv͵(0{_9x!wp\uR5Zna3S(PX7.US Sƾ2ħ=Ob&0.Nbn7Ѕ~A+, `q7tOR_{!^w?sth0 P`esAẂq~Mpb{6#sWi+Hp?^GGȹt[T>ҨJ8 XP6jfFmQ„fZY ]]U_)#odɘ YtA9B}禶ۼFD`XX8}+8=QD( X>&5.9=CXec8_!PyM@u CTB 6v JPG,d4i)QSexRtBe a_tlo(@BV:!g%ZR`.)Re'rʹP @g7n0&:[' ; qy,3Bv  " !Ieuv :cM`5bnxU Tc{EK{xJx:(zƣ kn^$ΣZ 4=y3Ӯ* k4KQ$13ҙY.!  B\F a/DEvZJ)}+]-I 0 ]r؍H 2EL $8&.#3đL _ 04rvOaI]6Z.> -gr.Ąk""qSpʤ9]IxO]1 TIpܽOTʜ &!Kc%r]1kYV!bc y./ь fbHwk&L/XdN\R ג"w$ B8 @x=oY!$ HM_>9{,E xXaFrh1kB㤯UP@?I 9QԤDA]R^Ad6#nO< "&nakǂxޝz7VDFlBxh␨P]4&uEN:^coݟy(e^.$FY04Op.E*Oט'Y_MF/"j~= 9m"!lp#@B jͮlu@O G`6J |Х+h,֒\?L\{K kC4!S qJP MR_/!)<}aþ[0|x D@҃j EMgjȠX+{:V1qj5*G0^D4Imث$ro_VޮM1[6v:L)~Ξ"i0 T?"n݇:)ht+P^3BA4&Մ}o ֭;)hhG@Bh:@mPK7aeBc$B߅T-RhJRr7SAmR(]IidOʝTr 2<شv:ۊJck" 09l:~#^@aDBIe*b}u a ġL*HH#S2jIhC$O~e5H6~/M32sJ d0[P-LdIſ( R(+V*]GmeT 8CN&` HB rWQOȣteWPB)[o#1 Jkv7c ׊MMR2"0 a.q&+IQWQp5T(tIg/$8^ig*xv5d>b~[K" ÊmG;vS[I} WQ̏^AC1+8RT|,%'H ٰr[FCEK)4*.O" 8!1^u0\Tj2juFsz1㎛Kja_nT !2W{RȸJ(C#ٌ<If2oGe9 UJ1OM(Ey Ne,aWXB xi{6AN06&M+8hzĦv@ aB8 K^#b&#ԉ{(Wr o*/Lȵ6!β`xCu|aDUChBAYRɐ"XBAiV!kmEP*vlA)d!w!M "pepIhx R>*c }P*Qӕti&HxM7ƒ*GTRҳ7ny D9fh;" 6!Du0TID2JXAKrr3XB;(U(RhZu R(cYT:t1 x(//|ZY) Y[ R~͖}gv@ ՄA$^㐜!DuڂMBJ|gAIP3 ND+O)řϪB!e%&cS QjB oKZL!,F 6-!!l ^ltJ@ Y`,rT =SBQ FrhFB XB‚2(c xBB,vP)Hp썤3\)fK2x,:"HiA^97Z8" C@A!Tc:R`DvV*HTI-IJq+ + b+GMBG FChR言" )oBbΒd.Y42oʄt޺ug"* HNd6O@?.WB\DO:'Tp)e)tҢ.L ̣}TEdfRJRIdO+)'sq& ¥@NÃl1Rcn!uP '~iгA_," È:Dp|nv5 .!90EqP )Q:78O`sZ1J*'HaAlE@9*vamc9?7jvu$[ %͖l_GS&8~@ !;N(W9ji1u:pr~7`G+3N٩c"H[(ьhR 93`Q9Bɳ>Z(ƙP_LeI mBh>8!5Webu+h{K0M ahs%7x6, K^gu>t)^e2XAYl* rP}w aSn6Xډ③BvUMr.>/xyq#u>X&De(~WnNLzwۨT&טH[;Mi9(9{@-*"]I3$:*]I2 ط2P)u2Q#%g81tQRV\5TWLlnA7FD`TuE:FVߡޟ,MoGOLzLJǫN`A>6hCL$jVٯh1O(bynu,-fFIDLg&9~s7n!"D&jY{W P:XιYcg&I[]PAP\8%լK"io:[ؘ5".#鳐JJ% 0n=HqT໭7Voղ Ƞو"PXAJ:2')XzZfeO3d5 E/e ЭU(r( ]GI"n -0qֻh<}ʖ|]G!n!@}Bh?6$(8^˖R9*\L4޷j۫ !a1$8z^@VHQdqW5,* WR]938)4\MٹnS(SY6'!!l Y J/NU+v)Npb#յ?9S T f$QB%)X !{@O&2J[ʐ@e}~#5V/47tX!! c񝈒jHܓ6iV)ʬ. z~r zUz!񕣐j 6̣DγJ)>HMȳZЈ!J9IfxBbjBf>Om1Luܢك|dMT "P`9jXWZDg5@u@LGqy6T8g;URa'HU*]客:jBdmA:fSYE3$@;,LG6ۄ0ltne] ac 葔tVR?1꠱ܘEA)=#|JT*QcX+'(W Uy c3Eui F* ڄ*L[c 0Ζx+ۀ2M`ð7Jڧn>{C6հ;"QQk:lKZcʳ3̢4uPkEt̡"uD3Y,Hr ՔR‚8E )NH!wN a{wԲ,/>r|Awu A6*,OB2f2RKWQ$2NaA EQrWR!A)dT&ȢvYGߴh]6eB>I;g'haWDH'-9ݩ\튬2Pg q+NtIu$faCu26!&b;zR«Ԓ<:9P6ZJ6#!!l~񫁞!H]PB*;H"Zd4̍L 31ʥeUEI E eP*QX(C?),f;wl$3W$"[ O!nD@GQ1!P!!Iixo7$zۯZv*9ss~ӲKae 3zJc>椴HT w XpBaoP!lrKBB$FL% :$;5]$ĢXIp J( lb(HL*Z">j,eRB2iI)4Fe|!W RaRȔŒ,/n>j<{r&{yں)@D TP b/0]7+)2xe[/&L,vIuЙs&MsPeYtT( 1jRa.\TtGݺfkxބ}M^ $m"h.}AZXDLRh(\T5H!',A:Z h+)/9)4Sp']NUr}B`FEf(w%e.nwcogB6K郿1$S @aL]!5xS%8.:訇C[wpVloarbU^ާ +'ј^'.{yUBeĞC9V O*m aS^ tb[RA ԹD'k, nEP 7B)1}4Jtf8)!37S2cʠAMY9 AUz3HDM4#Iak '.Γ>O13g~a!x "FXI ?la A]4X,388[pˆRZ+-sLNrILU`##[~n-j Z%ur IDAT Aa`GQ,3{xD-@z2h'0ޝ&IV}V5D1] _$#e~0 a֯:C)]I:JYXbB"ە8A&H!TIeByt%ꚅJ5+:]JmRX0 KBH㦛fx,߰3uDX)*쓟{cK@E@B7уBLp<1%jy$U$ū:hϻzs O$E(]z%1*΅[F"HZ8!d.s'%AѦ6$M!/gJJ_ɠk/ƦEgԮ낥2fi FEP \-HaAUQBJj$aYJ{]T. ׫̣R BH"\14B^>G/mG V+2W<}- O=l@ QB`$qAeHnj2QPcJjQR Ó|n:.1U$=EeO|s#N`ZV<{*|O9>[{Ê &X uCF;BО!YWdD.Ά装j]kTţ(CH "Pw*wRuHr\Cu Rb yU~JB^PBSW 1 _ 6+2YTmnp bӈ"D!+LB01^>Sqc2y\mHu+ѝYMevzZscq>d|;1B;6!!l+G:XX8˨(b/QH!BdDISʡL:cB}T02,:˨|OBN+i>C˶†ۨRiYxia_hp bӈ"BTe&~b-QtA̭$xD)d(_PlsC %QT+RU@B~.s B_Ik5hL T܏6w&B`CU. "Xk /23:J6BJPBQhIdR ٳB.&J,Cȿ)R$NsR8̢U[Owo⁃4" ^˨:8\u`iqF,;%%cyȘ<#I=7RKTnBs>)02X"utEBNA R&3+"|d( tPd3VrUe(4bEz&jR(3FeR]Päpsa۝mTh םDZ6j{4" qB8 .6EZ *6&C;N *HD2o3\ XΤsaٔ !lt'BB Wd };ؕ"蒠T uo˲*P+2&,֐2;Q^Z@*Q7 *ee#5KSH!/T.R9TG矢mR/?A'D!6 X a- \BQUO\rx ]5 u0TiO STVJ8v0fsr]1ud^gQ@]BEy !<+9 t:XF?VqJ(꫓̀pQPJJ|I2,ь"J<2J>N =,gZuB.jBY̞BF<7\6\($#OK NCl@/T\/!M-(:dzmU{&'~dй~b3݀"uAbU&uAzb2#yHt6;y+sCiXzGh YJt3QOBX a/ሪkj5!m2˨t]`)T(:Zn)Iy,:%T$ѪMuA 2;`/g0hսZ}jp*bӈ"&I ?mKWT_߇<@kҔ(xJ#, 1IRRkb-øAAcc_5c:$7?KS5y !*MmEHBW1!kUUmAuXTJHeQ^+*`H==ZJQXA U!{3T EQA]Rnx"L-k+\+6"6 @s;(%Bh):Zw5j:ڏ?2r-&؂ ֪]져¦ $ jë\I'2B=,b &BF?>jBG(3R5TG[6؄0b%Bi{Ŧ#hp*bӈ""uծwXr>֛Tw`{u0xUT{™g>(ΘmcXRxY^<@U79tweOh>H;X%?+ET݅6ֈ ո Os-@1!tiP LQPKmTBYVP%j*VODLzB%?FeRMy{lF# fB(Bz ƟLC#f1s2|QitW4" BXxY6U{E_OZY,+^WC;Zg^Jc1f.[2Zenp*bӈ"aQvBZ+ BAj{ _^"3FWYX3H1EVuz;lj2bXs u1$"0..NPe'ڋ6hFD3HtZ&JWSwmBo]G ;0IAyXFDBe S}ZWcgd?j21*ˋWݟqvw;)uIeWIjnUc%.ֲ:qyXf+O K׮lܡ#!ltBB ՄAwoO,N}xA R2UPDSrIp%UEEQ:y! 2޲dk=z{FDBvet8Aup(AZgucJ&{.1Hw) Ү[ j%.*I + z1R5ƥx !cB*T˨&(K&W0RZ<+z!L-봏668iD@* ax Y֊K2I$'aDm8A3ïJ4 >%:ȗ d.ո$\x%2:.dnAH8HMn*f}B>T*P*V :"C hyTDhF*sOu`gB;-nLNLC58iD@pHcx? yWwu%jt=? ULd i!uOuzPlNU=1uM1Bd@{\G$ 3VG4>ݘz\ź iw:]$JQJ0̣" #n"J5(t{Uvb`ncff`.auVF'mxc NGl@!$V1^:rP_PW"Fn웵=W/kÙ{{:q%Yu^25PUG9SyzlFfT&J!Y(p7RN/M!2ma0A g=$ >4"`Ie!ۧ駂2Co/ O!VD1+f󐩃nۄ%~kq% #pW:Sk/]P@ʚ2 w)դP>U<"KL4 '&sO9 `P"!l9ĦD BIQ5ʁCXe8ZvK%ji{%W=(^1nuJ'8/Vu0WU4"!HQ.U !,f6WW/T ){48cfw[H,O374jʰ;?Ey !<B9$IV]S ͈^CQ^6]FrDhy$ )' /Hyj!M,. >4" !PTԮJCX(%kjR8\ !qsC.'%dhV1Vclݙū?q3 !lw'g?՜:$#=/Ze\A a3WRcd0rMYRV}^*RX(= 2(]G)T"3, Ge T/Cl@%1fLg=/_ B8R c@ Vn&@{]7#!!lx앭i] +8>r .>}Pk:wBU^*u+b y|!\u&* O2Q}sX`XCl@2j1BX"Qξ}%boW B珼VW?5GfQW1!ZAS?(Z܇ Xeb-xsc6%rJ%k~ r3 !lwުMQ4")#x\uT uUPB]P)JHE 0.!4-\g!l9ĦD@2!12OCKp17!HRZ}~vFul&I(;; Qxf}a>I~uc7fpBc4.iVNUl.\$A}Tee_%WhuTd#u9A4UB::% uAH| iD@O/*8.!DuPp"Z?Rbm<\Ϡ 2S!k+UlVc o IDATU1zT{T LpGvYJkuy !0fsBX-V q&u6 wduBd!'\,DEߍ*#)O:c,{*VhyAMr[I^Gl@l* aLL: :s`k|-U~r8 a{PŢLJjnTlc~OI{q 8{!UܮD\+e B.ZU\C as.6a<;hAV1cQ4,wiąR7 +X=BT 913kh=,H&"A,#-A+1 \gF2s;rىb`iFՕ[ՇzA"Jd w{@u_mt.q|8i{KoXps !l{IQTCu\ wQ!)R>CD yT஢Z1$=.J!6 ؘSb:Bl@ aWjW՜ϗ ap`^&Q~!\fo!W̏$CCݹS#11vZs|6ⵏjHC QƢ]6Gm%&/Ջq̢^%ٸJXZ*lq UFP.H/IRRM aO!6 Bre&v$p?%GT4 s.D{[arv-zowJdƅmg>B@B8@ݦlp`uqb_l`-sKnDwQ#Tn4LdDT Y@H!K,CBHtQlQ^l ]F|iD(#P(2(8ax%kuԕ.`璸+Tl:]}۲ݮw_Ke=/Zm\E a_ٚB'9>بJtHi1%H4#HWQ>j vr+h @l@BX")^ބx8q ո Fs2X%$Ɇ/QK{#sduԢ m()%}/n\E a_q8HZInV/ƂFn.fQG\EugȠP,@gؘ3KwUT|iD#p1sg eT栅謕`dqX=` 4&UWXY.:1sp2 g9C:7rb5?rtz-R EQFyu>sRXt"!lĦD W:4UQ7%P+NK+F#/qjr[#;0n>AI3 @~Nx7kj [Y'vпX[gS9k w%@(N 7 a!6 ~8w ujB'e` =2qX}|ޮ5E"jjPy7և,{_!\G acp + X焰p^DBM#@;9(ɘB,qt*x%)Q3xqs{J~`bsRxBZ׾Vk/>!j%B_%yw!tDzy{N+)?,:_㩮GKR{>sT-)^[o1Ǿֵ!3nLIlEssAB8$rҧL0iA{j(2:(V_RB+ֽC2"س~gbe'e˜:}%v:Y,e1$%nkШ>)%cFX_]B򬕵ꚟ/uho  Փ[ߙXI(i 8WJUR+A2I_!DX 6A)O p^oZBePϧxO}{sҘ7v36c9ؽ9/euo  K7>o3,w6 N̂a8%]~\gjpZv+6C4+"H}=Xq,U;`$9J5#NqӞ48:>'2)`c2<\yNP$K;*ڷk.}c9{?>q!E"!!fh'x>DzeV. I֪QP6 0"XJ36.!zʪ'?@)8U O,uhAkKTRwBA:9 g_fbY?G>] M6B鵾( |S dȒY~*FYpjwW,L?+{](j*V?W}҆^9S _!^PDuО&yN bW;&`!%V !_gdDB8CxՒhMY8>,5"IjdIVB*}tHj:c}]8S {ڎ{{HaTM@=`l՛#FֺwL"Fޞa!ZNvx'F<%Ͳ>RZ#g !fj,Ȓ7⫨Vrsb>AƉ{L,3O%v oT GuҸiӳxUa$+DT&)^QӇΘ:;Cs]3d7'(>GgsZ!! YBϊqM~X4$M#=[K𙵕G+ڿi]C1Eyv6ˊ.`yQߝ:Xm4H\g:?/^hHHf-m'@J8$O"XE]$ !!hOac)(m~.c{[H|hҼuzN_2},n&Voax?ԉ/[uIbZBnؙ}~ȧ%v@ۏ=봌rC}jAFuQ)^1VrL퓗U5v11Y3vAcyb,3W<6⫗n<h(9_|6 !8/(&.کϪkq7WFݝk :J}ʺf%v@jUvՓ=Wb1w:G"D,I2jBV.{Tؘ:XVyLv Ut(ЯOv{hY\Tp%ߟ+ G.YCxն ڋD_RQ{A2:/d1ITy|׼#]D1A;tγڃo)אqgqISWl8h(KxU`x|Xnz{{ܪYIX~Bk|n,h Ek3&ᢼ $#2"p }ŚZG```Ә:?$uMOq꠲~}ʺ8"S cǝJ b٥sS9Ͷʤza):{`j =5Q[d9b(yo`?ž&aE)䄜ū1ࢾ$#4iig؉ITT`Y젞_䡝k]EF嫞IAJ=BkѨ1cZaڏK(X ŪzrU)@v^#!qѲs@@'!՚Erw,u% b&ZBk.%>مn#pbwE s}T áOx(}b8I^&ΡgRkm])%[V:atI:Kfgs;Ftn; !)%OFIAJO&), Zb`젚>uc>Bu;0+6yNO2"eă!@G.Cu0ƽL4?iJ6ZϚ*?Q0>ܱAt%g]Z갫!O)};فx@]XǶܕ mQ b0l+ i k]Ue"  ]>aJJiAc'޽LW:h>uf& ÈR42rI\u&ٗY ^16} _9==ǺF'L_b#Vz*Ɉ D%blN|CGL+ "PFM3}`TSa\,T+e2K N",:YCq#nOVԚqBUIW̹C>٩e_y|+61eo 7L:ؽ╴a8D,%X$I 䵼X;b=V ꠘ%@'{WG|bD`H1X }5)d :8?YFem]V.'Z0ca{"0ܷg@ dR2aL &D|=ݯO+eo7K}$Y+MRݼN^CJc:7LƇ^N |_ wC ͋[NnZrЗ Ovx[E?V zg䰵p + @p̞RD9(PrgOJ xNUuYkڕz?! rẃ!s,JuG:wUK} 4F72¦_vYx#(_K`rNW"+lԀ[ y|+?kţGoXiG1G{t:+)[۴M-{Q2ˇH⪵q3ͰX0-ܫ:GNj:;:2k="0XZ~ ;O+)A!%C6V{^Qg^ɉs?ɑpQDh NYRwK#k^ ?xG^OcSP EHϱiX&yhy|Rw:n;!(o?!lx$@B8ô0nt1-x)#drgOeUxdèw&w)ӜuW}ffUD`ouCA744ɞ$?bXS=?U߭[M -/ PByjv3㰎= $y[Yɛ)+KWXם$}tWMe X9| gsY,nu%yU !@czǝf([E_rFUƔTg?_i04T ݧۢ 1n[ !7P'[G0ShcK7p䇰7}tfՐû !'PWTW:X$X:h5\7&wpHtK7|:c +! pqJ&ՇaTH\$ũ9Y7/iqKjy\,`dύgK:>GIcb&AJᰘ-96 ARz_!ITe@B8 0d}`gvGdoˡBa<ΰhbۊCP,fCؒPȒo)AcyG$$CJAwP Q OBFnj~$uy?݉]_Ӣ_]~7&{l0t6O+ fw s2=9I7Ow.ۑ4uC@@B.Om %F pp) u`hHK&~g !DuPA*gg񫾲 Va<8?xo@"wwKI9p n;/+uw D~M5~][5" @ kܿt#6^ {.>a#ɝvUs;lJ&7L>cYXسxd {c 5bk%- ~߇'w bx"0H1k}=33}D''ǵ92'a3KaD"T ؂,Hp5N'y̦"$"$d$@F)H^2;rezS)'$sf?~sKU" @@Bg@嘊8<:pPFB@`w dhX9 [R\hXFXeStW z1,@ss֊GsAD`!ANm Xd(YtN-h~ 2Gj$m""pXG7n{mS7GVcw\{ʵ;}!PӸDT}:K _ԁB@R/ 6Sm ,JaBH >ws " QF! g3?4Cx[n]0։ g#NA'0"0Fp2KeX6PWc ČLcz"o֜ #pomX-z( dB/<0uPAMO8Xt]ɀWF@ Ξz1 (D@>"`7u3gZoN^=@A;<*oZ!>2ب:I E?c8D`Q!pQ 7B!t (};h&)%Bt]" @AX[,vЉ=t?7{`ݗ\jvo/ " !\` kk27? /njq;hv',pC+YG״Ļa,2Ȋ{YkNqf_}^"LSK эaX" Hq:  ~}<=!KCd) `,C(L4T .%#P :dKKj2X[|my|3?G'~xAD@@B#pQLn}$;-_Ȭ3 f"EA3 qxxc=}<- $uПX5J[W=[XL/ D@,@y  (G77Xuмi 2Gx@7s^@ڐ#0a ؄ܐQ8'8~@D`@B8dOZ! C}QeS3ȗ8D =:CtJB Bq7nɞo1J)U@3_t9?=$Lj"  !LE ?ky/9Хidw.<\B ;G}_"-(kOb g}8.꒺dYD2$~ް~@D`4@B8ㄽ#V?K)eH Auv=c3 "02rԅ6⒡(Fdgr DJ4֬}d@Ď""t®/"#pաW-u˾B W0u%%̢F' '7wߕG@ QBF\Bsr;yD@F$;vGRR@bg!G5+i)=dpȧ v@E ‹GI!וWuOu6Bz'!#S]/>ѣ ~@\dS¿'R'\\+-(E2S@D I@@DQc)P`;h MQl[7G_7 6(9C<;JO {ٮځieez;Hh\Lh1#:ۈ" T$^)_Ubu׍Zev\gtxD`tv+1C)&\/qT(YmD" & 7 @"H!D@B8xRED,B;a&AN lkc(ȒɠftH"aDh$ "\웏f7hI@ -5@y^m9 S ?*7eϜ=A) Y>Lډ|kZ"񩃼<\L) |~#" !\Htڈ@97.tRGYw,1Nb%a$h*ϾyCGt}B/4|q_er$@{S"bs"Yqc `f   pd ;I,IM&ӷ̢n`I,mՂE& )1%*Jp/cXi@$@PWWʤ AsD+T%` M:v,&_g̼eƥqӺ "0 !\\@='!+I!DufcJjy̆\`uV=ʢe)1Y .ܕ]CR !g?:_U/4D@G !"G6籠:&жg}gs%vBl4ƚl nbEXн`%B0 ݌}W `!,aP쎍'[XɖA6Qޓ{S]{:s9XFܳڧ7gr[r eZk5zrnGl1)f7kP @y( 8=:ʚ;w=UD`;h՛!w8: 6\.ƹwEW4o}*<ÂP @P@QB8`L^SAzu9^* 7EGA[oza:ʢV抧L߭n9PXqnwF54H!N޸BYM7qWݳA_Ew9kKλ\EX AE+\—?1ի1 _w0špe'jv9Rf;wpH1$ʛ욓s*ܵ5P @V@󋫃^NŕT0HptJ+&TUYʓ΋pӅ_}ck_pZ+TŪsV@8gAq8(0FЩ_8+֚_5q;gt :L+L S쨪*/+h߆^I8]lʝgc0:2v3]m#r۸m'a?{Ǧw|ކOۄs 7kBՆǎǐ߆߯6k{SkI3}rۜ| ̱2>YǍ=99k2dXenmk1A(r+ \@on>ӰK]M@` =Q9xO>z!<.??5Ek 0@!AC7>m8"(0 ~@!@( lPh64#,zP9ȍc~sPX;}Tjѱ-s_[dWs{B(P ,7q×jO ,L3] AO #5N>!"@!bO?VCv;qw fw! 42%POB] cU\F@%NNEewABrNr(ˁ mT((  DNm#7=`s/ҽV IDAToAwO2[|=YpP(X{k @uRM'+6ZP(Azb#rD3yP)4Rs =/@>O6(PLE8 &ywpP mPG)442Ew0BH 85f*sĭLOCapg4ԇxⓏ/{9oH~ ×&3iM]U׷8Nk @(pqZLP`xǹ8'Sc_TSÀVyѩIqs.fB$}$ϐˈ)b"d8qg j 1Tmaʌ\hC9Bi i%Q'G0J1)TfYL`.+ *EHI9B:ޮ -Phy/٭?w_'̫_t$G`XP @( y0v(0{ۧZ+. 1,58|(5cĹJ<7аS(#'CюBaJs _ ew*B2}/*:ik,78P7O´M^as+Xhfzy#o*Cs,u0BE?~Jso|<(1 XμelVƼ r _CApSHFIIQi$\qf PU@cҴjeri#L 6a؇0mKT=YKCJcCBZfżCՏks]n j_7RDt~!Лn6Gn>}y5[_wˁx9P` _O~9'jA%w[ uծ?>@X`nDR4mi!!՚׻238IPr)wPW#b1` vC) `(yGq?*BF'yQ7 c {8 0}uRP4e(}W]d苃o%~iKξ\pP @((R#RsqCOXʀ;2  ^(4!39ܛ0CaV9GraqQnEe (7PUvPȠ 8 Pr2:w]wQ/  w3٭ѫ0f9ˀ,!cw[|Fl`(P @( WtbqYPK[κxg6_n.:y  s#%%gOdsX \|fG7OsϡXJa|/Hu/6XB>oBq"ch 1k¤O ~>;0M!Tԍfs/,;>)ݿUuSyleC(ح**pYae7箅TBQŰ*pb\kz _`&~WN!+t0wYUhFE>6>#i_Xe kI=nJa'1;{!ʕD \<:s Na;8fkпOI|pzҟb{]1h|zӝo}MK`P @(D h0T(0On>tF+S2*C c:O?s-ZZsyBnB+%9?ZtP@${h=CC!c @ӹ]LEJ06A#&!pLC 9i1pwɜ´GaņP%u2_z-ԛp`l2\p,(hP@[ )t]Ȋ׷'.Da DpT$W%xn";A?$V!*R%> 5ߗ(Tmr֙75]pP @(@ cR^흉!e !6,F|@dX.jE -PO_y(RA ě\F`GބR-(Os0*LcERիT)` m"ڇ{ 4g;>;!'mLݞ/ \1LHZIdG9qUs9$]vʽ T> J'c\@ AaJ |@(T!ZR0=bay!N~.* cyT/>yTh :? 9\EH3R; 3BvBRz WM*h X|~aM@(ʍÔyN %TN<=  'z[U wxG>? : rhv%kh!GTPGaP;P8 j"'}76䝟ZAVcV[rಹ~q0( qk@5V}g\XWX:CE R$hK@Or{qyv*HsȦ)L0?~FǏZVĪȌЂSH-%o1J(7ώCG 9RJBaN== B.bSB7\~sVUW;VR.r5^pP @(@ cS31v2E@&\Vmv>+4#ݿ *UKUuQ iQ_HFBL0mLww%PCali!.>MPPci]T;R9BJBaQ Ȅt<]XF ȸ\&_ }C/B#Ox <|ݽ7SN%׌mxP`5⪠@/wu/nʾs0ʢj$ת螴+;0ki!p =B g*Pm)pQFPIT|,rP3= !0ҾSL *HX+rSy-A!c҄[_ܩ\jJے 2ǹ@=zs9zpsw%&sSl$mytǙ;aۍ69wo亾xe^[0v(X3W)ywN馵PHTK]@B>v 52rk JϢ g)b4Ja, ܿ"3LBB{yA!rmU0f뜛n\  T@@q*(06nڃ/*`ru 9i,Hg`GUyҠnX͹Zԁ@U(!:)5{W. Y 90 2L0i7Rm($S fd\>ߚBA!݋G9:+Hv8ΡG{ TQSȍ PC)~9|B=r%@% @(ptP`L  t!"‰txɃl |rTB |BNjKQr  H,A $\I T.)x ` oSBnWQB YeP;~ W)` ( m%P+P8=~hRᷲ79XP`a &5NƧ@ ;kLoa LF)S p() ^)(n? eRBK97PBCWpNկ޳3RCuXTF-¢v8w1L~N B{y0-P`@-]E[BI.slot0F:4&/LCch*Q@ %w0@Pă;OtC(JV!-Aa2&PH΢e؃C7 #ͨ^*TZIHBgT= uQCx|U} {T^&OE9(W:d4ٟ BC54GhZEV q- P`mTB@]26S!;I B3Aʢ m&͕,Ca_(,-LCCi|D!*[ T)"eB0? #PQuRL2QΡ"ʡ8?LOle%w0 B,P @(P 'RKJ] ~ 4cxo k%ElHNQMmG#>94T] I()CCA6Bf\PP5Qr\ ކ*",p ar#пvy~m#ac1Gnh nskј~\%FXY+;0(Э@GrFjڷr *k`0>*y4<0n$W1BFMTG'ᐁ>},)>f3XBhânPpZxh!0ܾBz>x9r:Pȿh!{P @(0/RǁK@p fZ]~=tz3Ȑ B3'PUP@HR[TԫR -t'9KyuvӟU )\=g,Aa~]F=HBCx7ݗ=A*2P`Y.aP` @,x].dS~zΆmh:r~B5#PJ\Ð/Eh| ME])$1¤ /UIvzB<ЏG9`|}%ES癞-'- pCU CR@rm'sX6 . ZPQ9S2W:w<`Z_`F$is n|OBNs1'8ү>Iנ[EPҴs9OJB]C6TPƃ, EW3 -P(5= =`^'?b|L)CB,. @(0z"  iBm`04ʅR!I_pZ` q ت C !UW0UJ905T`RSϐPb5ITCvM<@7,|4uׅPД2¤Da)CM6MuzUhƞ8iG[hNw0T½[pd(➀kB{rXpHA(]TGt94A7E4sQ?l M]z!5ϊsi-B1H:)#X+$wTq)m02(MMZrrck&ҡp?T9H•r[C)Ԯ.BH{`LaVj3A qNaZ}4-(C xi5Si!@Gra-|\HfK{mc Qt IF@x; @8ÀP ui\'((p׾S[) tA,2Y1gJU-$؀sc)k IDATݴ lJޟGn=(+#%=q59zj?q!ELs_G5˜g'9I> )A!9PU7)'vbP @}P@P`, !LkHk[{ʥo)2m3!Zꀒ9f8RFmtƙMhj ɭ-$ʦۗC0j)r 9 uƬz@q2VZTP!;~cw|ˉ:tcY1(Z) \BTgpSxqA8i%ӕE}3΁ۃ'f(T}vS*9\1\} )\5).B}B0 .js5RmzJpQ uE9B*F+Ԯ`[7Or!׽Q,P @(` cR wťQ $(L ch) P.^كbX-4T-@@vEE94@/SQϟ@nSN , J p 2z?p;9cZ$1(( \I%B&">?mVh/' T=D,.ݔ;ȭ}u[lN ;BRP" ,oW u2+TS@at$@/Ƃ0"~Qԕ?UBQnAb3%( 6}!7}{N};V}Z9*!l(X…ʍAq)P%Xlχ׬'u:b#W.8j|K4:8y>jmzD(PVsc$9( !*0B:I."JX ;VUBӪ!ٳC(窢i~ Gax(9q9d roǵJb4P @W@s+  4Re"m&xH=L^$ᏲAsf; `^G l"Vi),K៬qW= .apl ɼw3 &Z LKBlG`\>>ݣ;x rN }xneMTn rD cP|&,R rC)DЅ}ߘ'd2OJ> _!Ċ  V@`q:(0&@w0ʢ,R9ԃf)PC 8Yة'2EhTBNnA#E@F. Y[ ]|JSwJrhhc; nmaNj|QWaɻ?Eٻ-/5),tpÚsnzcZ+0(X];2(ЩCBESxJrr'ͫQ5p8*;i$BOd.( Tp&R{H#NbVIQG(PѐKHw Ќת!cfR Ħj(|&^h?Aܿ,rAƻoyE) Ee6P @) 8X>R z巀XP,ǭxZmBRsdxwPCCm3VEuQX!ه21LVߴ{GceҐSӢ2\%(> }.NiP TaBaN(=O'ܖpyNIe>5po9ňKpiW–TqֺW&\JNrt X{1lJa.@@Yd.\Y~(BPI!PCa e n`\ 5i*OphFW0N=cs0dp [CR ep0PhKC ].gHwP\Dvtѫ_T<+g%?p$>w̩3O!GJ&($EiCA Q&N$/0#pQr 5J $ʮc8Uum,T!2: |⮿5y'֝窪*P _#{}SP wPvmuGDsR J(ͫ:$ PZ]a P~{9iKڗ)%CDPH8:)*Q;)|¹@aROczx@DlÁPXcza ns>dni wuSL&9 J 9w6u$TTJ! ؞"dIkڞ+r{ >ɽ U OП+R>Tͨc9 'ͱ{?yKP }'P !]@ؒ;T`G~TU.mT2<( qǢOG!$c7 L3CGUKDH*0IpRȩ?@^oЗ@!\ )TrLmd95(EmvA.(ۓSs샟2>,G ڈQ@(I:6 d .;HjSa q}j}xg ʸ-PSLrFp7PL{\AZ! F0kDM MC*@!e7y=X Nj`pWZtj9.ye]E(4ShFcGa**ؓTA8>/Eic(7C.44UIjh*9Bg(LɔTn0s c觪PD kh٘ Ч;; ĺ  U@Pq2(0."N/A˿ V =mRu@%!54Iz;q,4C?5H.`>.JʕE33蠋\@UF,&`B.N)1Ἴ;6qK*rs{]lr:swؽ\8uTͿШkLp\+%FXm=:(Ъ@+v|*;Ak<9;8@AiH9Lx} "4tߊ@1E^"T-qY(XƝh'/gP¬xk**RYRc^aEՠޅ: }]٥e%6P ,Z@84(^2]( rM Zðy|w0wRYԩqtà$:E3P [iQtR[=l |r Gma+; LB9xdG/ #y!9dWOދa \?7iVO+áI3,ƅ~3AH  @( aqPAF \;_%ɛe^sS5bπW_ ,kp\:ks_w趏Na15LQ( ՠP;Ȍ/j-.anpBuhht԰^zm"0yB3 RXh@#9? 9iMIVU! ܱG2*7]G/C rzX LSsϛxH2A-(( \0^( Ä;ΜE+.Ԥ-6_( _@- c;sP ŅXxF`S|A@9` LE62!t! +^q~() \ B@;Lap8+V!B3ڲh!!xЧރ r"b/(qMc1Br ʢ\iTZ_Pħ!d}ۋ|NiB,P @(P 'R !yjʢJT8 5 9}IN^ܖgĻdhXpсZ-$w0 YMbbZD& s;p%1D2dPa=0 S0!0).2DB!6 lCkhpfh5t;4@ ṟVaNv咰2ũ 9)-IUSwىMDf6kAA i& J'CaCBp1 U7pzNCVT~c͉O: |XלĹC @(prdP`\ D ^.m1lcĢۥwpH!| 5ȇ;Tb'":p8O/Ң"?2&o1)(=9<4->">C"+ر+m%(t_VŤ?# @}oMLuРJ @( W|qyPMnZ3 Dl8Kxn]8^p2 vosHu,KZ5wPYS.d(L:Mx^Zn:7rBr&,g{&߆2E&P`Mp rJi (@JM>p3P @5 @(pъ|P`D  ٴRe @%X!ZR C_wPߠ)T?C*Cyʶʽky'LzO0T:e)r)R)_+/FQGUc}gK@r!pGHb(P @5P@KM o{qe& 9^݀Sw%\Ah!zU.C`H%\3  ?. ~(>v;/"PcL UP8B >9Ggr%bC(b.Vo JF T[A[7?s% ՇpPNq Ƹj!t@2 j`kp &H`r h`plF#f=kP;|R%T#1%@]g^i?&_/(m'?IS_cvlsrA(XEs@*PwPb2#q; p0,V46ڏc _|T8'B~TCz4'Q0 FǏz9q d|^'9*7 HDD]hىvj?9y AHK @( Wufq]PU. ,A=ojŜp<}۫9'PHM`* .UeBΡv),JcOC+8]hFPޟ~PR)1εra@N?B:*1j?jN~p=\B#!pvP @ K(a' Q|8p*_6am;L0`|@6ۤhv1P,IõzN HKHq'pP 搨 ݚ?fvr@8Wg}ŕ@(E @a@*4(C̮ 5q1$KWW,Cxtw9'-}zAJ8wb>wW;<82Baȉ14'eP&*؇p#v!;.$/>nNր,g,t8$P * X^hrZ jZ8e) lkv+Ѡ&_*$m:5/9zGz;y} ;uQDհ~j/>fN~CFY:`08&h;2. @(hV#R?҉2t߰xENCN[E: 6n:PX]I#3Ũb.7cp`UkNC \mF 3%cxL]j(w3.=F4>zSsDž>t<@Xm@85P ue\#hP fbu I@+N[,RYf9rR@g_:*6נVrn'!x n-!`l/H= ` l% B3~ZkΗ;\C(.Zq H:,:+NI j@8Pd3VcwPRJ1EKs ͤ۲C N%9xƙm (BFGFb(P @uP@k 4"xd6zz9^j]%@Z+w8Zq(y P!Ќ$E0r7E)JPUuN!W"pUnPo9h[p\C(.Zq H:^ 3wpoVkjGHugt!%yOO #/ٽCТCP ڨv 1$E(ӻimCla8#Z#1(:( \Y5B^a[ p-_/?P^Y=QAtR;Է{]iTI.9.Dz mI!ax*CV=,r%Q}ސ^kbv ԘGcm@ bP @+ \8!ܹUj{_Y`a]ܣ khtUX}1Q=%r2 H::~2\J?C ] zCn3A2mv\C>z8 ,~ѐC|>}FpL$Xk0ɸD(Ф@e' `_[|FcJAmE싂0TI{픣[0.'u|3 $e%cB pBǟ,T-@ @(pъ|P`D  XLV: 3@pӅ-orۡ4 k+7 +΁_sc9m+sI9-BnR? hA!p@{L?Ij- @(DL4. @wJvrc\8vzIWeV&X_wz_.* }XrH0%NŅg p* Q*7PBE} Ir}=c2/4r9XP ,B"T9HaC5 sbf*ǰhh~]Zt;u+ES.\ 8-9U lq5L1!-:>b66["V +PHs˽ jaA(VW- t*@h\hGȟJOJ2=.+g`N xv9^ &0q%EoX;tIHL'bEtP@!4QT*JN"s5;4#!dt;vCعra(ycA%S?U*S!(`):wIMw ;HiK!0m.ʮsr+ f0 @R2̠л\\)&5) ASx™V4( E5VD~@w;FmyuZkwK++B;YUI x^B<1Qz>QnƈKW}q^X+*#/@"+. @(< g0R(0w<馱?''PQw;^z &T*F(b$J ؗB3l֚78GϒOv=ٕC8P @VA+ #q{ӕC)Ξ,Yp<ʢ3 b%G/H 2yW0X@nnG%GP9z9Za>@w_pd\:P`_8)u @(N{OexI»smCXf: 2WӧۇQ#^|r4%T|CBk}⸙>´ N{I-p#FX3׸R(PS`Csv8#Y*R";|'PFB=E(|̞71O0q 3. V,=rfbP @* \8)6;@␵>K\e\DFɳweQw\5Dsǎ]jhz:0[qP0QC(g o8w>9l/ECh j`pWZhB,hA?C{A&YX"L~3&y苃 }uB%cbW1M^Yz?i;؇tp׻P'ኲ9 sXtC(.Nk NYB;Xa9ަBr|;{lqS! _#$?si7tpi5vYA?!Ģ U@Xq6(0*@87w pRx;Qq0I m}#Z!&Z̨U8P)de_!K:oe^d7 UURvpZ1(( \uB/liCĎz?VlS;8#:Ъ4P/ZAOvޥps PȽ ߭L_F@GUqTnZ٥)!j(X A1)PԺ עR^j!ﾼrju8OE0!Cc)%!rjZ K@CH @( aqPA{8^bL4U t/w*J&*R7jb% Cs}Qu=v7:~TC!43'ܶ"؎ r޴Vm(X…IA)]iwV*w#d5PKy?F> 7HIsRb+%zAj@ u1c4|46] &ُ Rz2hfPFpºoĈ+p@h/t#|Nr|85~"dExNwqG+6,Wl|qQP=4Ԏ pa1!2sõ=S~{ @ @(prdP`\ KEe$CpP{e3ʢ]Zy"<'Ms#esSc,q}XsOҧg@ PḖI @( W~qPY@`Ґ˪C뾹 ;yT])w0׋kpA! ;S9Ԟ(Pi>i>@<hC(B.Tn KdpCA1 gq- P`@pIUa%CGhb\9$wPm YrGȁ0=D꾄2Q!05Ǡ?rP'w(dT= gbцP @* \8!l!_eCeQCf Dr Ѫ< ĊsQ Wיꚓpך@.cYAA5!4vɿxQynsnP` 2F@3G,vHAc\={`:хfhccG^bXP @UU@3=HZn!;ܲEp|*%3[-E`t||ԦP$? $Ow}ZC{PT&2%yxo{,] @(pbPP`"A=h{uJ pRkS9AiPk7Z-;H$wP?P}<Ԝ~ Sid'pQ @( }1~( "Rۉ)IrgAYί Xs==TFmr笗;/.MVc^}6.jbϡK PC~]/WT9Xְ+P T@8P0lVIv98mp?*=\S5DzZ*}yU-w鶷mZˠpf c{@1[w4tY21m<jXLh?kq0^BQ!lsZTt뜃2G BP` EQ;C{w8;[K<5OċP"X9s` p;UΘ*ZTH&b_pKuin;?!6B;uP!j\pd\:P`_8)u +Gd@ 5=6!` aszu1,vy+@Xg?TTl wZv|K#̹ ]QI9+C8P µj\(+Ph=Ywrze,6Y2n%bC^ZE:V||3l;s ZIph2 h2W'wPpnbP @) \8)9Qm>hjwʎPǫ8 p nspFE+Qmw0 Am-j%SrRx8n2?}A^J @( WzzqqP]V|/ 9{`-- Ut<5$?vV W,j5{ܮ$](ޠQ rNB,P @(P 'R;^@6@n.:A5$LΡ8bRF <ǵLb4P @W@S  `X|. ;e!]df7ĵKPo?{;^,Zoc Ģ  U@Pq2(0.ZpVtZ{V;H7L[|,.Hy9V( y[M 97 7a/wЩ 2@(V^O1. 4+P!yj;88#Wu(|QR'A= - )P 5F-f^M6 @E @(prdP`\ aʢ{v88^^`{WE?Fw H?K2:^Z&1(+ \)Bbw=DNЯ*SvA]&@az)y5Pcb um'*!  S@0q"(0>RP!AEVZq{jE Wkۣ^bz %8.:p]!w:O+ #s Ẅ́Z2/Vp?spc:nߊA(** \Y5A D ^<~U-5Oh ޯh<`>{ݥ<viAiZ?($Ai?ivsqLgA ^ߝד{i`@sfP @(0GK@vw@R\QS`KQ_pڵh,ZnWbZg{@@8cANx&(MD!qYĘ¥\O1h(* \֙øp@85;؋^jɱ3hvx Y^zݮH:=9`pislFiaƒehaa!P`Arac(Z ԁpQ;8M@3`x6 KB 9ۀ`ZCiRb.P .簂CZ, @( `0D(W +^<5MSً0jdP `<7ΐ;UL2W  E1+wp.SLChb[\~h~l{U/xjY9vg)4K'beVKP ,\% x@8lBZ>9 A Ɂll ,}:   P`]L:@AS ^ɱP-`Ԇ8%yDfNk*ԪbP @* \8e \nN.!p];z#Ɛ 5QaEZ+eZ[8(l&=Z#1(:( \Y5BV2 pI\ʢ, ͳwwpM-!k(XE+A)PB%+;^J#Ϭ 8MU!5s1;L<7hPP`Ҡ@@ڻ؃Tq]%X%#,UU *Pw0?ןcZB^E.@t IDATEkP @(0oVǃK@ sV`;X2!mECa߾5 ? hP+pE&fQx_VLm;,'YigpRLu1f( cmۮB29ˠmP:/vA/BFgYʰP B.îP`@N + rZg3!NتbhuBPQ9\޳7<\s zuޡcuՒZI%m'Ck+X&4[+]@,:S+',1wS;*zz 0Q,P`tb@_Sӿ\A~lHeQ,xCew>wM^A(XEs@*poyN71/k;ؔ;w0s^׮Tʵ1ƶb2"[e*p샿wH @(VLM(. Q.{&.d%VzG~W*.rg+,sya2μYƶP @(fU@8r k7Me_ė3h|188" zt57bqn[!f\P ,%$  wُʸ ɐ5_R\;Ak+T4A2:߻z>8.P @Xcn?WcpE.C~AsV˄.7z~>VUv묃x/Mt(XSA)p+~2Mc͗/ЙCZ9GG?$LBXEnhU#G=GD̅*Z)3i 7`>'uyNt޸C|*c>m:c[/0(XM9*(Щs`O?UWc6C0` ZePҽo3@ʢ_=hHh-sZ+t~P @9( 8XFgm<9^S<ϜgvvNi;튀34wp<.P۬W\wZ R[BoeHBky~ꪫvqmP`.|aP`n +ΩvkJ3bShS* YM'iHp6'vל v}qv!Gxο.RTuSo~͟<<<PA!n ( v߹afe˃VGGszhߕ1=râ;Hp P]3:7ܷ. @(pbTP`L  |s VzON Z A|Tךdk1 P ՜W\T57+bppZv?(wZ4TT}_y|[g|_t~P @]* ܥ ,9>YOga%iFZؽ5b*R+c[g]?XcP @R@\B(pW;;N9kSRεA1¥CP @Q@"ˀ}5v22SeQ.Chxw ctFw-$5 Y|fu7~A(pb ,ӎ_Jc)R;gg~Wa `̀:o$ȵUx'7UE13a۞;آW-TtVn_dwڭ{[xСP` G>A.\N7mcj{i2  Uvr)WR j{e0w /^ڮ*_'XP @(X!(F ~g;N!=ݮ Ćju;X+ڲ2`XyUU[gRKP @* \8O> ~bk/h/p)󮜤UqױN޹pk\?pn(X];2((p~5I'[3ฝU*ڗΫW~W^Z 0*+r.,N:oTs.:­g[Un[{95(V@8oEq<(0BЩ'>o5Zr38E3{fNk/[5T[g|#\^0$(XrK>>/ܰfX?vu;8 y;(o1VֽTׯ=_tѕEhGMe^{Ɖ'| P @ **`o+k_ey60M9tAm,]Zk'UuP P G|tQ(8`L~ŕ3wNIGOdn ݼ]w!K\ǜS[#,yUN[grP`).aPw_pŷ1rp)j"B?ԷaTV/xnٌEVU}N/9%b+(H ,ӎcy1_`; pjPџ`Uv9Y]!;WIu5e\0f( Fpp/s:kS3c)vouN[q)-@8'Eʹa0!Jj֫P"̵ڭ3VrEA(B.Tn /ʘ1gȶizѮlN>vH{c}8<*Oʢܮ dpʝ:rwPrz;cUUM_Up(XeP @ )X.&jO+ƚ+cn2V ow0Zsթ꒯~wZ|P @(< g05W`enTG͞;ەJxX~2毫y1=?5>\*pܘn|+;αƞouMCR{Yn|N9t_\qrZw#K:6P =Pa 4b瑳AdeVUDU/99p (/ p7NL#?TsaHt}[pR(( \HT;λEc~0D\,ԱE˰¿ԘFcmyxM F |>9Z; cklXhXc޲q};5MqP @^ {Ʉ(AXu:Lm>VMe~w_BU36ɏ[TBc$%Vsrk/v> QN 0c6M]eqQhrkwlT?}9z?@w+ |߯8v{˒pCw*wPtn,n 7-P B P` ~nbS2LYjq7ؕ6Uu'pcHP`n 㢠sNV̘ʞ_Jbѥ4ˬzl9st @( p0U gA{TUfc|+>ʠ);9k+@L]z5ڣD/v*S]Q nqd(( \0W_v} ]n=,vqXBS=/=pמKԼ1ivPT\Qr ½z @(0z" pシN_ _㦪SrO޵pfO+[5>]_-9E8P`l 6#Z)`]pN;X/VvwZM. ,Hqeͯ3.`AHP,H @(Fp$aGܸ5# z.9,vO5_vc'oFpP` |S[_2Xr˗B2sTX@bo @( } 0uTOx'~sv|ob'[p$yq' kQq1XZpљs/(r-)-v}x2TP @( Q|z=+woL2sRt,Ne֚ _p=G\+`_zO55cɈ6w0&̎5_Ku0(k{0ο&ƾZcg[w7:DA( 8o{Tk_`-\t;RȿrO;mk/w^q3C(".Bec+vbj5v=a5W}ѯۛ_Wy=ƷX P`$ |tbkE\l& -~CfXLY}ʢŶ4`z玝^r;xH0P` mGUtߛ1fs?CE CDDr! 7D"22B@@0%OwZkժO9ݽCsv ]k}kհjV{_p䒱 `( s2:h[Kwy7TU`>uЩQ kK֛%f{GI(tҨ* Hɦuࢅ_/??:?+/Z蠃Afs`T#θ9_u"5%G6;w S* ki)oa.{~l Oila( nnU` \3ﻴ@iB tixa( wxpߡwnUTUU` |yydd~o.|wΞ ;apeLt IkBhp }`nn?^=t^WTU`P \75ͬ^p{͜l}= F9OE7sӺm(\^^|8͂` 0$OR՗3W_%7.}l6:*P ̭u[sx[z,OYCĭqtPapͻf  tG' O-` Lp+#>\4U@PT5Q@pMdD7~p@tt e?jk}ΈLg5Ar{ _xo-TTU`s)@Sk |:lNcO\v;0AV5<&D &`0L IDATa1p#}5UG)A4kU@PV DG>\1mΙwO2:`pf2[jM)N%:Lci|ć-$ZUU@P6 kgۍ` s#k [CY* E4US Baw%ŴA#\6~Qzw:{GK Q@P;ĖVl8{8NGZY0 O;Qn7G]3<%C'`2?Zz>PG~߹yѵpw[뭕 **@b |>}[8#Nkcv|[h0,*;/E91գԑP !v7> Z 1ɢ3u4rhvwz5mWTU`(@QZjʹ͍݇~f=8\a2\t;*|g>:(Bme%_"ZU@n} vT: )13W]3p%ё_\(f R@pK5~鸏~,=pp 8H*:Xp.O'wޫoΆ[WPTͦ7~h,l+_Bz﫮YTkƒ6~+[\BfZU@g%f >;oixO^)/,K +9dXk_1m{v4CG SV['?DkaX(FvՠS^9a0_W0X7X5hYZEKg燤<)ݡefXмMu} e>Yn.-ʐ|ڹ9v4ΌÛf]FWI 6%0C{A/Jɬd_ysގ=O;6kSV5ğ_ 1;w-_iȡ `bsj{X֡{C* ;Rv>+VVχ *sx >ۿgjhe7Y0cY50֘u֫hpЇTABٻ.vi{vJh T;{Vui-5\!H".<;h+w^WPT|Ș ua˓WK"FRS>VykmI 5~ :j%-x@8@l%*"}kJcnC/W:fu\T)@8LMOSw;xhNm @>@zٲ<dž3oX0o1g7H*0Y.8ԹySF^`S?;Yx_ur`jֈlFav\u[jUUU` |1O8,^`}[6LtqB΍hH$85xO'{2[Z;{Ӊ!mtLHwz`/^Dߤ (n҆պ舋>d/S+mjFk0ȵHndckyZ϶Hj@Uo=.0NܿӮ;ʟ"sns= BUZ,Em `cj뮻JV-Mܗ?m߹cko~-bLdYlAcz4gvjTU` ֚kħ~BDKpvdHkYbQX q2k3wRPO~n /Sɸ `}02v~ z-Ibc.Ңk3;GKڱY#^K$ u`rwln qb/Ž/ͯ_UTU(pé~?`-]0lltcFU'Psz女C#QtI*W2xҪ ЊioiD%fs\u{whmnm@IG^1 cѝCfu娜h0JKkf0D#m/#^™.:{_=HfZ-U`O8ɎKO&V3T\E$#Qpd(3h3ɞ>?] KR{ M6<1^㑱׌&F,)@8K1Xx`ݎ"&Or>S`xմbPNN@+f%x.?} S=,E `kFOWNLqҬRµTwџ>bk- wv]K|#2+{̀[/N:I*5#H0vѝ I06tJS)/u%+G?Cf4:Xlʄt 6*x4Z5l&huѽyDv{`\޺eYc9O&ZU@!yoe??c?0ZQ>5T]@i5:Hw>@yR?x.oWa|4w07C:C×e P \Q3OS Ƨh.JK94:X2\ҥq͐Y:_ hᄫNz'KPל|C,]0H`!2UzFz$3A뀽Lˡ5[HNv~kRw ˌ7]JJ ;|p;c?^QynG]@0` &`^Egqp V6Ct/F}VV#ʼm NZtU@az`Bfo]sTDt it)DR<#{dy,jmVoaMBW(ܴ'/xl^h ݌V;Xހ{7 Ho=fl{ /s{ gdS^Xb5к? ^{5:ĢF m{3dܸ?QGxga7EP ]c.#X8'_ASP6|Oh/:C4:HqWV]ZW$M$P4 ; 恗q e=1>D)_VWiURɟ^mҜP>;?&t7@[撣?k/P=8ȠV@ R0`*fj#;~cm"Oo<{Y ,zCʢm uՕ?q^I8`P[+cayl=u7]FƭS# ֟98_WG6|3lnfsшsN{onETTM\~ UN:}7)DWP3t+X5w OuO}$정?[^m2!i5w%g? 7]P 673lwQSV4r#]jx7/)nSTMu'?ƣ ҁe"^ձ1 Kv!KKf<}GĜCҕ?³$!ZĖw~=%m@39#c[ S_YOJdRѩk-yo>WH$N 1p Xxؼk6髤@8M\qo0Vj6Zfđ0+XaD 5QZ.:s!v</)_nESTMտ?}q >Vg$5R+x]XҖ }~tx\´צ$(umӥ|^!}jZ~b3a dp(Ewh!4jT @vOEpGڿ1uvq:zs(/rwui4l{Nq˝N콚6hFp̿ҜoarVKp>y]+֌kmὃ3 ر1C}U2R XkN~ܳzլPgzrc/i3]y<0{`W 8 oC)}lۋ~SlaeYO}ߕTnG&:蒦}E<҉Y"C\Q{rTC')gpM(8ʹR N CYL ly``5уC{!<줤#?e5˘[{i(WN}#DKːqSdͅ~}2ֿe1F+Q VLYPa{'@(GإI .9 n(~83 (LSĂ|~ff#{N7H⁁lf՞,ejbՊ zb9{ |'HVQ‡>t>[#k:6|ǎqm4qc4Ev~ E^zagS]2vdm <3D&r{)`̊j8ာǘ4kHcXPBY§8QTR MH7779<¡1RjDIHN46+E;2o?o^|W2~nzcF/G~˰c+ p#:l9A^ S; ` Zs@@-$EgdE{f3 ]X \k']{JWr9'wv-iybPf>Ki RXvL@ m"hBeć0݁6?K'G$M߯'[jc]?~!={UFm&BGsf惑6B9`%Ȧv]`B; (9@(_YBpE-pSJg 0.0ށ8ˁ=l9ׅrƍr<"mJBDŢd/?'ak++Mp.J\ۢCꆨ 5gSNb`q]Fx5Y"\#zu WZU@آ |86!b%b^f`v_rdE ,G۳'[p^@Ȝ[Uk~ 6\tlQVe/92r)!7Պ.gO܇z3mo(qy0m=+2p fDgia;"aeAYK0kd+tB9 d$¤ʣtNض"D'2L!voPb b(Ϳ%wA1?!U ]: l9 Fy=S AvsUTz11/˂|Zg 34xgk|W IDATK8:8^lfqb%tA0S6\dP{L6DZ|xAW=f;j1TU`+pݣ &݇i{L&X 21|{!KspAu /x?pm;'{r_|kE X2tk%,Gcv8!aXH(_ ͒g/$@Mڽ/5p{ cfhU7~_7~/7g_3Y}h.-޲3=I˙Toawl*  rXt6o˷f'l(?<3 ;tAPTQm,GrPE!b`wDw~LsBaO{Zi# )سN8B2ێA+OrIB_ܶ@h̼a J@XpYO|>@A&@(EIOi},HC5߲-%}/ IW"fd̫|TS@pdQbm4g/U#AлSAfP9 AљhɼXx'EPTU?搳@Dg仢Ҟ&'=BM5tc"̡%A6 fl;  CYh|nJ@"m]KZ%m0=ЦɃ@yQZ~%7.aD;K@ \;uUNutPN"@ +z~ 4WOWo'7PR@pqբa`|Biąe/7bAK+ #d;|LM`vXn~eQWTUUGzwkv9Эf=Nbl +% YTȮR ,`>ߕrokr% pQ(&#m=j/+|za\IJ$@(,M'`X {A"͌$,#ݳ+,>x]+LyC|% ]?u-9? f,ܠ WWfHPX>L4]jp@X-n(PʡV@"x+l '\qS* _#mn.dI:EjOu1 Nlid3VhK%:ֶ"V2G | r,vɴpOF+LfcX H[h,!XBR BG3=CI98?O&nO_cuJʵ.Wyeb90T6A+hpb3{٨΅ l~]>˼ajry72}75{U@PZ.|C{kN~Ad|4IaHpK6W{, 瀐 䧇2%d FN`T?osS=U8))Eђm@)| 1ҖAH ìoKe?(R8}Ae0#^KP}NI?7`wѽ3fk >Gqwpt:H:1+μq&:ȟ׍xj!/'fK6!b%'U(gyx‰_|5S쎚* Dk$&J=0ق ؛ۥr FE/9k|l狋#:؞pZBYt0mP+eat=0qM`>qINV}?~/ӊI8iU{' Z-߆< l] =?R3o/oy9; SlKacwC" 3=(,^}QÅ'q-{a6{q䗟(ɢā3޸9cNȕOv  %{F+!u+qz4 [`RHq_vS¼umn ~!ʕ!ao$!GL\ Oˇ, .Y?(BTs]dڵ (!wJ`"}׺}eA7,X3ZvKt؝ s;ā=*@8LIVI,'lzi dmKPnTpof 9y)<jߕۉýI-amzfޯNi?3XH^EHe08॥5{Ł0\TNcp(K8@SP< >6KLfPT22G)TK*B& K,-=WI~4ˣ?zgC)%G ck[^f?7sU8hFv*l`3)f;6\kpf <vGɢ~ /ZcV In`CF{0\&::0KodGc[3B\_d<r-<%~PQR!Y~H O&r S>SnPb04RJ]*fgv{ϕ.巌F U0B SlOkɢ+x';%ļ-.]̬hw,GR6Yɢr>]j֪* +pcN9BxMՌh<v€iOeC}K1X^7ˊ[zmЕStz+쀥JVIOA:MC8%k>߯_B˿XȲJi$6 o  D73 ban[:j0*AucxxW^3G_`Ѹ6zW^=1v*Aat})rgP{K˷,=S|5kU@ \S|]`˒ƞJA?~'0\ԮP3OPr=tK2a@| /0 FhqR[PdҙsR&% QҖ2ʼnq&!6A e ]dXޣ@} /WQvA>sgoy9 u SG}rɀKf@l`3fq -#5q ălvYɈ;m"\Q+*<@8ŗOVT>攝`ۃώ%&8H>5[M"4X#ΦJ@clG< 'XS@&)&,DJ@X;X CƦPw )6AKV]!':7.B(|N" R!u1֡{:І!%9>ۃص٨~¸tI/rבoۡ6,Wꀤ_XSTr9nq`S)/fQ $ځ6:Z p( fg5 lk߸)* @ ┆,ꮙ9q^jAdnhws2N*_~d/Z!6hۓ3';j@('N)zFr` @~7!JwAMXQ;2 1!ݿ(gٖYw6-z+ +jx톁/ݲw<] GQ][\y>)++p R}濑 byQH1eX.—뿕l}҉( Q G^n:B r&G]ďsCQ/Ia1Yz+  覊bYd?gX)N-ԬUU @ C+X9$ӄ3J9yu_FiX*`0Zk̾^eKT2,˕'Ș۳ǻںpӋC{aub)a鵂C0~ہ+{m-!?8(}6!*/% Q5a#_0-G}NkHU9*WB<Dx?Iu;Z[!m k@8ŗNVT x 4!LNw&06vKE LdNL(3$YWȾ哄[N>_;$Jd'iղy+Ce `@%ZƘwٶxTT=[B/=dG ^NzM|ߊƴxOm薎|拐OyLN3ctЕ\0ރ~q (N\ p/)F#^ZZ6hP6L .7UU`@:Ӵ\Zhr$Cw KOt ſ]MUPHTdf`ApMBtZEi*ht{6:8>EՀ9Cz[/iOqnpV`k4!r l;q% #&98'R KKCJqr?+2{޼Ǔ6|Q #t69>K3>.7퉣HZMXsG/Ĝ͏5] V@Tb t6*6l~0k 1L/4+ƖH$SAI=:3!F'zi*J]`[2%81\ 6:% , O.8x%v+G9GX'9([ԙc0b5W(Nٷ+Ҧi}X=C V]M-@)W_][Bt+bf}tPػ +KҲ[R݉V#̀}#! >U+}\pMӏ bG ʊ3>Yl]9$$ 3Am=Fk@8IPT* {lC[4,wu3gL 9=*F~,}Q{aO(L龶C(9l&DGϒ+ Wp"8)/tKaٶ K@,j斃3^sӤ>H_ ]XL}uK~PMͥfd+?QK^ +Un9 2ۚ6/wшw*kY qF)@iF~zGFoM':i*0aR b"beH0nʵR$v?`JGSȔY]jXm'@Bjs(]60=9#0os^Bg boCЁXy9ivxEsL7q׿e DIS.|M_h{Mwunc8}FW ,̿j4 (j-a4:n \/jgu(bCf[g}kV% "L µx4MU@X ֎#Ռ9 {&V Bd>!GR %Di[=nH' iHe@v  /͟ S۞?*GhkbB,4R)EԿ9C+ d"oߔ?v1fj4)@"&.dqEuo/ȀdX<;$5V P.8@ILOa-`\cYNTTU` P Ec4>\61η[lb I0U#!um' S*hBjDwv:N~@(¹>%@I0?Dk*!kBO0:{%^gRB{F~h'wP_Hزi=̾xt9q1!M*(@8 ћ/\;K^V?, 3Y YBW vfC\^Zrq&(;DrCh@8PVT'v(!9-  ~O[@̙q)y{% Jp@wgWXD‘]=Y>5m>H@B2įy}t_z M',~ݐ'QD dw 9 \pxof=WVDF6Xo͍+:( y1=n8RP[aīE rB4VQժsZp@8PVT"ݵو@D|bn㬃fb0A'?9,qdGD|y fq\g,-$$@5?3lVo͊ GJ€rjxv o ڄOY vRJGŻΨ}C `08EwcwuX@(i|9q4wdb$MiHiz;/므kr@ţ ;qq# n2:PVTY!` ]90Gn1"s+8d@Ӗ98 KLe1̙f}bx iE98)a.㶾ÁrvLv2P q:?O:󏱾~V>T<i?O#tx2 11т9 _׼':x~0{,zhOfz NoYrT-jƥ兓z!GZP>t [ؘai>ihYtzo  yyN2ցGRMv|t!@Fp$ Å7\.h~_t fKzQ;U L H@DQE]CD=})a%ER1~~N-FIu1Y0XDW@p59a:E/\;Rf^] 4@_;壃7hŽ=;uGZP)`Pv̝N'-,?K!#.:0m,@Y'Ȯi:iEO `u8(g'0/O /屉aiE} {_]{^@?+*=sDci?k?rg…%V c\|phs't,^_c Q?N6̹H˿ԍ8Wl ̏:)vGZPTӀq{'=6^̸JyJ h'f "KŲ%ItyOE m|" IDAT-rwA0W{'h̥9rO)NC.Gqz[ e"Õ SHNZmf4 DR3!Ȃ12[//g`QVTU(@4QbtA]h06F;FUɩ!uɓfcxv{ v$@'⪙tOv! 0*@tl_8%2 L#MD&\4b{}Y$v43|@A(4wҨߘ{f\~pԂyt(^_#M,0dxg|HL/AR i7O;j֪* 8꒾&8:^;2]1|ī/#gLgU O6sډe D0>"*m~+ V^j>| ~<@>m1266m۵aB֞E Wn#?+C6eZFP (+B:#nd"c- /5:H|,EtmԯM@?0<`S94[)NZP.BhɨS)-[NǏµىp<4:Hx>"D tl$)Ut؇0|HDvIk |D0_t}xvZ:sF]# E $ZМ|>˜ Sv4a یJm~wmwW ]1BA [c`7t-O+j֪* $ ¡Rz+1rh ::t]`B8ݟS–Ega "Bd|}28MjwA&ɕ<KGcm'oeɳq6C|w߷&B@8{@8}D<'ؽfuLAh``pÍ_Lt;qW)vEZPT^@ˎoՍі 0KF ,N? L(A;^2:P wʟWy/y>ܒ:ԉ,J oH`ns|RG4QtZLNaBhpScQ -Oƥ垒`Wח`ߔI +N)NQ|=-W-Dā7(ɜ<7`j%metA˶p޳6ŮY*( EW| I-ԍ 3ɢ:HN|U%HbrpdK8"D9>;OѓEvh}_i>y ]mRO!;){6!MaXSX}ub'Jm%ῆk?eߪ!Ks!V0ߞh(~֬sB}f``C5i՗0_Jz>7= pApf{f @‰D9b^508v$8]}g g 03٥ªmHrt}ԃ7|n&b*^JʁA W J8b`Efc~xLF,v`>Q vB)Gdo10ؗ?. 敠KF` rџptdž8=6 ƾh, f S7(.2\6րhpřbWԬUU@Y4SٳQu8` kgihK0t0Ej;~fZ T,̥Hf`,2Yf}=~{=B_tvU J@컠t)U*IF>MXwçJF! u 'm: C%qB@8U+@8E/9/M0:Ttd\څY[C26 u'K$lޱi:ŮY*(# dx*4 \-c (XE"mgG#teh:>}u\Bq.ŤpQ1"bP-ZlLnk ȼqeJ`ێ :ʷ -oȀXԠ{ʼnA:DC!8_T-'Wpihp:Hp:@(20lخ:.g Q9q( iՎ̹-.isVa8<7pg|]QVTU Q =:IȾv^z'a5H5>UѦξL# ! N=inew u{UD25P*Ej7 Mz_xMx9]Үo (BBtBF7ο#yj`mt~al(^-!s,I˯odt?*AX)!)ݺai2|^hX V2Y/3AB}kZU`K+U3sܞ3_wNEԬUU@TX^0v_, @`% lR#9nk&xLJs$`٫V lG$(`wj>1_lB; ,L0H5g+R@A->!+)rg[,琴cQ^H@ L Cأ~^1ᬆvɨFe|!apAJx":X>GA\\(sܕO{f W.bf 1 ZR@%aU&gZPG 'CzbВ;0]r8.a,#/ "Z3@H2gQd0mCb@)נCe@(˥@8E[@8E_;8ЮBً6#؅{)vCZPT2+MpIu}Y*!X9%h.-rm1h ~XupA8x2#Ų^@sn}Pl095Wd@7b=:V\3yrK@1P(cH.{|x@Kԧ\X!%R E DQ ׾Ult/>`Y1:ˠVŽSYm3 ™]OPTR Dfl^"SǕömuU^@<"-bBუT 9h:<)'6y(%@4>A9w)+{IhKZ]=_}aG)ぷv a*⼨?#)L6/ Hm|ӂ@8_p_rO`n rzS兛ĽOC\0ؚ[9:c0~l_ 5kU@PD@(N͞9eGg|ډ\"\5R SB E E G6֒: }0ۇ;%aH%Ej4{9Ѡ:( ܉l1 {b.Ae%7tRqOyﴹvMo%SE S>~8)1`3>^ǃAR%kAPZ+BK9 y0ԡZcV!yl |qWWS삚* Y(f&!W26P# .fdOȔb &( rF{F@@$l]z$@'v)L}g^pʨ`;^WW2@$}Kcҋ֟yYtP;ط @ꍯ ЗA׍1Myt0iO)#)O~`   _R z.?`aqDOb4*Z O j֪* {ڑAhwjbޞ ̃U "KdRJբ!5d'D ,ؑ!nA:iє(ඏ䮝:ZEt]Txّӯ:-A`5[>.~9ngDEG~xa&~]]lJT!5R ?-X?`-܅n齃(0mq942 47޲.g\;.Y*U F-:TFAccs.f,m~' F8B@X$ZSlDvD ¾`*l,0ѝM_S dx<MGU( \¡?gCD'_B3"J;'}³i_i-Io>9ʆO70B{I:EߴtFh(gmOhle$e_AЕ`\)c4|P9Eth`$$[i/D_3Y*PT@žv$ aRݷ%E 8wM03>ҷ\g~0 Y'vB[ i3,a /CĽ$މmcM-$\ $ w})˖|0}\ ^"vWh{w6 D ,_sy?͑[ಣ/|Ye>>Vj0H GWpe.8^K|bԙMP2ygAjŴX* (dsD z?ZP \#v] ];ccs &PL O" PmGUBa4 H-z9:g~b&<`1@~}#ӳ\B_ VLjB% 9ڄCQ 7+EdKJrh..Z풢6d N$)NQۻG7^o<by6ZmQ0KEkQǚ]`ue@VD L" )w=^PT@:ŚlDO Qdɧ!Er% ,Civ+# ʜ◔+S2KV6!PqlY2 K$Xe(%-ZNPA TXR`kʹρp;"]hYnwޅ"NlU85X5 L5Է}%uQ U@8-s1o..AJ@H?e#F9 >w. );=L x [ݼ%@"*U.@A?@Aaͣ-4`^-rvUk-[uQBvA bρZ|-|Ze>'>%?! ATOA"Ń;^f+QA@VIQcm&?]F|%}6mvj-rTTwɏpв2>nR/FHW4X'/gvy?@"*PT awĤ,Qx3MWeJj@H;l{,Y\VdUW#zD1 E@F2GH+_P222vYR-nwYMt4C@h%@8U@8U]>C^|`mp}f1:Bm:s lnq |s,ǃ`(sAb{[sܕvZU@PV,Zsy$+':y%)l9GG$BveVR$= :@'i>IGjU`h+xwgr2wk^G`rm#&PJ0M1 6 2 A0ax4.* q@@hNm\ VCT-T叙_zw+(~4JKBˆ=TG0|Vhu ]53*  Bљfʉ5,>awh1P+/l_.(EWb,垗"Q8e\;@&SAi[*T)rJ g" tB7DYQ9x: Ámȁh2 r+"CiH@hCeo젣@8U@8Uc7tY/}~Efu`W9cn@WZ F:6Asŧ~gFCPT@(DD| ՉEKdVD5 >O#|iՖVzL@-ZCI*v‡&xu9 =`k!"e@(`=T.')&/`1T|2mZtOC|w$ִI{J!,aM̰@8U@8Ui柽ۇbe=2;~fSX߲Vr>LuJ呅-λUg0CM  @| wfG!0:6%C^N}v͛SN/,Bī |!o@?/$ID5Bї;m镱t~#&޿bw6qK&9燴5 rP) R SI S_s|1^rx"vUpڑ;6rJT.PW^b|^AKDAF!fc>3jZU@P Bc{t,94ӱџ-@-:B_#X$NtxfAa.I0PH !  bw3 txxY: E C]m: >@融e&MjURt܏8gA'TtQp6IpKY7+/G ͑З{Ej0H82L juТaT ˳QbZH3uzziuxq+~g쟱QTUBxZqa(d"ODx1B8,:mPPKmRSDt4dФjUf ×I ~maJ@`X,DH塮HN7L IDAT+Bː9۞ٵqLn"t" gC+ J@ȟ {| #y \EtPpHpMQZkϴ# D4+6ĈSC&`5ݳ}G-{,pӯnERTU7^*_˽ 8$UdIFlpn蘽!%wr6yv ;Mp$vCװuA 8 L`'-ޟ@A |6U FoC|'-,kq?yӉ9aSF1V#CԚw'.d114KuNOp` EeC6Aa,+kD3N* %.B( @(C0] >v9veHVYw€ 'Q*DMN"sLڴ-ٞ1==ڤmì'0?Qmow\T i/Ekr;7}D_$T)a-XcR !}Ep]U5?1;Q{J@X-q4JxP6s8eOeYJg"%b=HfU> STi N{DO.\ dvm+eFK!a  `P\C7Ol.Yk?UB@B3NpԵ` kA})\a !f 2δ>!b3|6t7*Sit>˗R D#;Qr#(Pzab#NC$  idRVʽD>,/$_MHAZ% 6n]2ZV{ 6_S \]'ef}Hn zIlU]kf1:ӑIgpIzbm+vlٱ̷Rщ* LAMd4wfaَp +ezʄZuh&cN5;a_mDt]J5&B3cցE(5 ĀN ϩUaw0RxS|&\4'_?߆墙<| C{S?C,¡M sS/e; `0Н:_ ʰψ /61FS٫,IF gAG˭9U;iASTTU`ձaJ@lR^I8躀%oRZQ蔜:(9m1YgܞGuYgl;3WyRP2z-Phaډ$t⶙R% uN!Xt%N@8Ef}.o ֵ?5:[+11&o@y'~̯m[* ^nadt,L[KpzgV 彁 U9 mNj#a'P `W܅rRu &@0 L'%=Au ZoWi">pO,OS>ޣXG!&]‰K6 ^SY\nx2WQ (Ha0ldqϣ(bffI}ddO/Ff_:sk4UU@PW~@X LX^ NN% peaVz^^+aJ@Xm!!D}1ݳM6 I 8NB6̖Nmi#!Go!/==TWͯ z0^ӊ``\Y3 ϔAK5+qj1/I:Y> Q[c҂v~<+M ̠Șpyik* @|=X oGp(|t0@ʃ۳zw x;R#$7o?pQ.A/j*@{Y+9\ȴܳsX9LhtbSYIM}]E!7xL%[ ]v% 7CN $8=9 E}a jAy K7j 763CrC"Zk᥷ڿL5-U@PA㹳%x\.؉f=@bG~8 蠋PH V: ;<ضLjt0 PZC\{1b]]|r;{%!Na p}4"ʿG=ïMD'xK@{@I~0f[2HžBF5M9yo1{6ff `/oje+4,:3/(Cx"hZZ%M/-{U{6& (!ȓLѩ`gGpNM*aۂZt0FWTg%ro tDBX.6괢's R#x@Q,p/)|%~tҨ̖ Cs7 }.@"uRp]\f?m){lj[ABlbr0oiS{Ը*οg^u>뇓kUMIPTQ i,Y2Z;'{aU&t_^E#Q6f!sڳ;e䠕GB݃u ~hd@}ypRlߎiT"Ĕ7}KeIt03 YO/ 9y_)MwU8N`)ډ) SY7PNx>X{~uj!-i6Ó4Bʃh\G^AW^wK0Nϫ*8~H mKAtO^̝_j%+lpv2 WZD7inröݔA QD6C70[Gie/ (!BV0nRh' J@(:2A ߮O-WyِKdNlԛ`uh/a^`,ܗ6yXLt,+q:DkRڋ136w0[߶`_u-soxȿ>Gk9}ZPTVQs!{ڞ/Q l4@eIϘ+N,#ɨ~Y DLJzPN4RB>! 'aў6sۏ+ž 䯔sO~˷9X?ƞV>}\Bɾ=  Ց\g~o5@];1!L#TspZ %fqOODZJ>وѯD+tXlZiOgKgn{'?{@K d03 e>ueG:_i&͇@*p ^.&mM-IfO n45֎:wGG >ifwAb zrqAZl7j76m@8ّv6RS vH)>{=xdE`@f2w丂E+ {=MH_DTU`X!ĠD ls^H P*NSXL],.0_3Tf_?,U N},n>p0=|ē՗-H£Vf8w"Xed-k>1B϶&cI4I,m'v03Wjoq61)a1NPI)lx`8$6?@At 7!lAf3pl`G ]Kfp* LX 6ȎwەP#؍-a!ᇜ#z @xSii(,ҦkH} ŁԣC+ x HꏡN.8H Ix+BCgЉeHD)!O-5c ᄇڙHNp&a>ﺣYm= 6|L8$/-ѐO4(Sw4kF8ʧd[HSSTU`a/;"Ɂ"XRcGb1ޞ H[F4-ȩ@|ns~BT+l|pX} r\/I^7L;Nz ̘驺8=湸=U#?v +Qm<~l߷x츹žl4P%Q?bpVȃX0|0Nlpfd4,?9~u]fUU@Jδ_= \9עCrC{R{^w5 De^W@sp\4(5EN=hw'(iK -$YBZ܎8Y+|ʨ]}]K5 ..en{B4Ҫ+G|2BxuɎQpzLMp`m=xG2,k;#=&5\yÔXrpf9LY$L fi' ~ a]5q$YIKpVZbqOm7RT.5`LdQf| F|` :g>ml_~\YPa#hE<8l!D <[ ݈]E DeW;q$B F hA/R°?L>?.!ad4)?)=B AsU~KZ= .] }?=&N^GO~R >@!T (Fzs{ ˣ?0`nI=A-7lȮt 5y|l4wس޶Z~U@P&p aG1\BG S{]t0 tE|7[r|!:Ȧ- h9cb@h/ (/a\APNzUI@Ȑ9~:}S>W1RzG*LaQl2?(|Y B}3(Nrg%ֹn7gnHg[0'c|}ue-#/FnFs=;kv* l0+2% DN}v/25KN?Ԯ{| G_=m{ \ :+}m[x cu@H5oO{(7.#]&b'g n\ L| <2u߬!1 rti@!T (fK7GgNԶS .5wi>J_@H6eBZ"bcPޛۖw}kۍdcDQl!d0BU$Prq {%ݶTa (0)W966EȤ.l, _O߸>{ֽ콆Zg|Z?qik ;߷ݿzp;8(Ҥ~n)LoyMmt 'Uux͊ouL5yNOOaa:ӓwWD;@i~|INys l_,v@,'UR8_L*[ݰV; [ajJkMq9i`26}vǍ9@(%w0n1f"ת-Y&s:崝jyoKb,t/{7UHKC89Sp!቟9rvw~J 60 !صkw/͛7-H݆P |~W0UTYP`aNv?KrvJTJwm;:Mrz?'7Vced0\ ,'kyIL3EÚ}=4+[JlFR[@"B7b_~{;r@x ?[^~CxNNaz}`9&U&[fwNO~mOl( @(p ;э  3VIgʊ# z G)Cymɕ1Z Zt'UKBjf00f8b_js fW~=l|X(`E,!3* t:H1tQ.=@xz}o^g>o:s8)L=KB8L᷆0f{vP’NB4_> />r{?}׾K"P XzCo͡d TnYw0 zU(~`50wBCgF^4'n% IDAT= ?Dn.MYL #Ƚj;/rݣ/(9,&(JOHJO/6 >,V!@N:ܴǞQf稧=~mj!O: <ѿU7̋>g~o|n /8}<^i:NípS]?䭧@(L6Q.rLvxx/INnYӅ}5"!ombKɁk5AԷV:[4 X5igCÂ&^¾;D*&Rm4Bh;+0j,0~YJ< /*0~6,/Z! YԗO^B%H&Hg_D9BV ڌ6?ydLj!j4x)ZH֝@ÂFA(Z ¸ljk{@OK=N:^Jcw֊:MzYf!vp ) AxEAb,aoLVwHI Lz?b.H&+_UPo,Ӥ:^ JkiL)r] ڧ xQP @SvD&Թt>%&a \e O@՝qr܉B>PewEqd黃(cD "1Mи!l'w~8K/Qj 5ʾa8\g;0o'ѠeG\2ot`郞' Opa"(Ǧ LgSI;ʉ r9xQ#{F`’vhRSX!ݥJR rYI!;σ2kFprGs'S>9~Xci @( $ uډqVNL `?ߚBٖ-u!\o;h!; S1#Es{a/`mu.Xa-YM u4pt 7KɀЂA H;ZۃvLsqR4KE23@H#h]#ڗm^A!tl( us<:~2cr1h2:O`[սqR׌ j9RhP*,YmX$:wPhIن m$O#Gk %p&Pk](nYxd`sD WFf>5o^^t}P @(@ ='J8%UCUicNtSGqJ'wǰ(Ǭa/_j5N С@X' GpŁ6~ AYz7&|r\.Cx@#f@(%8%Axp%F`F$OIUY${˜0Ap1?GñХk)tEi-Ea'E J ]ڱ0;MCO”ؽ}]uvz Osz[mnKIPXkA,8wpl#?lmƵP j>ފ>+PIBVM(mrs#pX?l]> ^ d {{@w lOشm{m(9|77^%sǗ~|î7 C(8/*܁5Ў ˡ2!'{ivk8= `c2x@!ѪH- >%7VN~̶ֲ#;\"5IEB~"@aV,:D'w8oSM]bb֎=89P%Xw ƀ9.-mc tcZX,11!.p 2&zSE ]VD.*Y$!M 0hd` bLo,51I{_$ᘧ6Ո=G@x^A(8c 5Ƞv k[x=wPϦJ*9X/+R0CYBqɢM667CaD(ց1 E\%ڱ1@Hs{T‘ iOXNO3d3-髧N}t>+@8f+T.aAP;{3pg P @(pu D ~+`<)A5$ ifB ;RefTHPRՐ%hkA6L{!91?O$߇ɱL'Wˊǵ&s9ev|Cxu]@ 6@(1@7_aL|pD8;̂Vmr|-sph~J5D;PjEHK`+vgC w ,0  o%JWƁP@:f˽t`-&cPR`!tDzeG)p% P (@] E5z S~'* p/ik!͈@|~b9 pJEN?tI h@SyS wx=Yqp*d?HCrʨ-g,݇mnP @(p @4Aҙ- xqnJK[iUWv8rUq:ІBlx-tm 3>1Ƥ/AҹӜ55Y!lBXoo`iw,_:Ѐx9+)f|o9e4]ngt(% tRa >*:eÛH.aPa9ЦR"V1bkp*dļo +7L@QƜt>O)-Fy3[\?ܠ_*p 3>Pw6P*pw +`T&ٵkP= 4 XZP;řl\*`tmOhS LUs'#uЙ|3Raw_ormi# }Yj^^ƨ @()7}ruP:,Ar6YDȒnh8]Ho|(?ĥ׀0;B7IdL_̟i%@:@KGቝ9@n^Y 6Bcްf_hq(p8p8eaWx6ig2TZ>WDwA ܻGV ܪ( @(a(p * Gi[ %hBcw0M;Ajjw􊿏@P Q}jP7!wgQ0\ƄV{;BEAf@gr]WV#a<{_v;(P`cL t@UJBŨZtr! & >]4_ƼfD S6ʞZPCRѴݲ4y=^2&)q F>o65_J{lKYƗzPơv|| /0G\|?Δ=&W(P*LBF0X" Wbc{U@h 2p(S5je8Gj/,8kL9R0[a }Lw&-Q@F%\P`@؁ 'h [)L>+YA0BӉ.6K:1hC(/P~o/kS;i 2k ih }G_44IURM s$ƓINT+t/+ȱJ~C(8^a dGA{G`7C/quD⾤\ L*gƅ\ B; dSJM N샘  m@OPjX# \9ܧpI|d'A(=F@ c#m+!_j'yA =wЀV%.&Qw+f] jN Q&O(w0A iu9IKr%z'WL/vo4: B<Ϣ,('4tX:s0ʥ<;:xP@JȦ5_^ HtIY'GRvzZQ 5&1cKr%IKHGqj+?.vA:'r.?eIν|ܧR[RHL~a+zP\ @؃L׊|P;>ԇj6BcIjrQ2v6XlN)ut_ Kjr6`2Dƽ%7Y8|Lx/Z+= 1L%u]J-sBHb;.D+P!=RBE6{9us=.f!:P` |-_&t0F7 a`SJbR>/"ƇBo@ȯ)|ۂZ+ T;X5h pkBk;(.s tƤv+vsfMe`SX>$]QƤ?κ,Ǥ8}1uUTa~! [rFmP @(B׍0ZhOz'H*?du͒ciԮ2u/8cm aFܒ[?]ĽڶVP`'H@8!ti*0Q0X\ܰ8S:q?8<<8BJ3݄Ԕ_vBZRV3 >p"uT*;@ OaiC3vkw0V't}X"@)B">"9||;?Dh P @W`n \6}m;*G0{k)A}MpB?R,"0h0wYiJ+O-a_J~1&>UX- x1(!6 l%7-@\Wy2WFRCM @(@8Q+h/?fOd&B#wt ~ H+AR$h| $hW\k mƄB餹!Bt:4إ@(Ac5Տ98G,~.8x6RH; ܱAsP C <;Xd 0?OprphH'V׎X|ERK@R<܅pҚ(쳴VV 26I3 g}&z@w©Ӏt#WpɈ0pTB (P@)pRIh8 ڭ`E!wb︔l|(:yn! t#֖xbu K;zduL5&eߥ/[+֡?y?I-7P $5&3$i`!gT@xFpP @CVsy/`2M+q}R=HsSo.Œ9wr/>HP i({ 0Q +ץ0,]9*\Z1SQkU@8.彭6oOt*1]w +@xObEP @(R3CxEu]Y2:;|,1zwP&`\|ԡa/JiK0g6@/{ma'bAK5~,o}wheՑ} +(_vړ$㊈:'W4:WLojP @(vKOkm<!B:sWCh2jK]9PPC;m&LwdrIõ|al}<%nu8YT!isnQ4A@ξU@QK駐:=w9|b@\2@ GFH:P@xCA(8 7r50djyY&mIAOq,L?_u\IV F>JizsԞĄ9c˗k{jvŴ4RF֗z@X]9-ao8g] @()s1@(AÄj u02 Ę;6r RX= p%h rv FT][:ۺIM5חmh-cP='+M:t/~{CkUi>ݍP-]P @(mn :O3`QJVJwAs!zK8OB;0‡֊w_VB$r/'*,qO@hF6pt=+I{)  SxXP @(V+y%r. 2Z(9X@8qi!k%`>4T 6 Dhi՚ۜsGv: ĞT&Z,)L 8%d(CYZ++\xn@(8<*v" =Ȁ F<`SSjI;v  &F@R I;\[tMy {!<6BWOFUp=}6BzWqzycm麓u^)CxxzzDP @`=bچ׀W1xM;QڵH^:ѦCmګ!*0AXXD@3;XAVa,ɔjJGP'p@m6"=Od95SYFY 9޳@St^1=3h)P @(7B11vAcs{0XPCk z#zW:/_:ocuj:I1# SC)= Yىj KZy.d*\g&tXy$Yp΀pz;R83P @k!퐇"%qrD0,@*)nV9VADD,>Jȃt#$EەƷvSbz>( 4Y6I KAynyD @ѽ̦\OnnVuM0)&4G n .P @(p, P ܞ;XwP@ `¢űBp=VL *5:kG@h`i׆७l0n[p*R/t2-,?ַ̃&܇e ݝ~wg,(;'!샖 `a'4&rhH.vj`  ǫ }@otFﺍLyH a''a" m\hNv_|a8S1 o4>|p<!#lt @(Z@Ȃ~.z'كa9)q"p;w@7ovh!28T[ :E|= Ex]A$6Bd.V:{d~@Am_" 5|>#uYm"s?}Jm) ܖ( @()}_–v"uK]sj&MǫA%`*yh,:VZ/y 2q4@hݯO0Ƴ6 A6!)6ʏZ}OS؆KmS*f`硆;(٬KF"=#0^A(gU•jIK1;HNɵW:rmu~@h: WBy\#ˀІp%6 dU!H!mqv yE4VwYViV|_P @cS`s \Fw0ɹ MX!֥P} ǧ{YèA}X m.S`UµRP A:] 4 UiM=fF ']^nɗ/,|Mc[oE|qB6rD/V_P`/@ w0PXw0Xer 6̓0aA{=)j"us2i6׎"emVue_;AO8M9tڇ"##RC;ZoQ62깃 E*,-[^.ZTueysM>\ U)zP Zw9et'ZzNTp`34hOu@tE~ taK;FYcX42Gt | 'NNsZ #檀.{y: ^j Hm`VBUI^ @(+p`OI8`D8&:E#<$_! }(m0?VǙ3=L,wJGP @(vS p TrU\a<? VDXxuR+[>ŔVGpR۔\k\: R/+ew= LZJ{VmIlO@H:~$t @#f@(%@xw0FQAf~% _&1C#}[փ;O b0P`֙yb%|bTr>uk;(^= oyŏGUc bn;@P @(vI=w㬙By)ÃkA GY8A=o3ԀӠt;`Ԃp{*:7@ וD5EP1/a!HzNyhvL ִpS;Ot-`y2S{,GB=p+LtmkĒ5*]5 C(_q A8A 8BK",'zhN82L0j>R,1)]A ` Wɡ+ǫA_[te@F$GPJڬmMzy@<a{@JGP @(vS$I+o{XP NS&%| qt?\)+РԆxqwښNBoa7?&zY*;Oڑʪc@TL;|a6:~v'JT~TP`7pGkNtGp)Hez2ik['\6ȒC r$CCv;bs?\qQ=C(vkXPDŽ_:!<;i`#:9 1z,̓19ܬB q(M5tºfm!X|9]:=nn~ ܏qB+P \fi'd.흖ÖvMY&!NNcʍP.ÔZSKF3<4v} `m<%XB_bIjN2R ˈ1t,*AJX ;^!< B2<0ʇP @( H ܚ;DCXrR;.x{gB@(gIY>* /n%^M.J?r"A'r`ӻ~f 0ZJCWD%>.!at³{VLՅP @=Qy(Vuݱ]9IdÑ}!ayPp3P @(caoaY#i'2] C@L&c]s L>&:Z֤1Pp@Z@82KHYf{H o`1dN3Ҷ i.ӼY";CL-ݬScq5P @( P @#T_t!v(y?N$ $]% wYĜi6+R Bbm&GҎ:\rXH 8EQЃheYSF&3Z@sO# Nᙋx"lv" G}!P @(RѷFUM]]n'v{Z6Z90 HBJZ PmŒUuPLMi'HN@hhA&yez˜Q t) .leE^ {Ҏ5Z>ގS G##>P @#T_{a _vݔKU||/08{߲ T ŬWBt;F,i#A`qJIG)OJQ`TSnHNI0*S=#yݩKk|PZv[S#ۙ IDATѡC= x.6&1R1+/я [Uי\g4=De G ">P @(ph U8;EWԽml-;H׷o@o_%E2 1CP @o}͗OSC9sY2ᣐ\GÖP:3*-Üz%h0A`|B;KB gBG*G#GgaSAU;u RiډXyPvSWw@$Ozo4<8/{w:YxCk+`kzq6LJq 6{- k`~껃싁X$ݳIÉv~a dw > &~,B9 Sxwhxp!Q<<>P @3(0az !\gPGHX( kA;=Jt9Pz@eprsCxAs򋆦g>+9&ixvd6/4@(A:bp p̀psw%hI4StCBziLot!u@. /otw@=_1 ϪP @(p Z?L \/'Oe_T(3 @(0x⭯~na幇Bb~5pS/uzaeKOR{,`c@ۡ^6eZ|jݗ L;aF!Y$C^5P!=Fz`9D \\t)'oC(U_sC<$bgtTOL5 ^߄)O:KV5 {FpQHg.K^ A67 @d5יmmPb6z@hia|ZKa9, ;@&@(E7 !|w,JFiq!4J\B+HcwÁp3wPB\8gY!/2h)P_!4w% Zz/(8mڳ}KGu@ẖA|h@0 [.\]|K2'P @3([_iމi/at 頽\#@ki/V놥&E`}a ՊӖIˬSJ]ֺ$U!׽r]<&8QR*_чr3<=V ?j\t! /DV P0xsn0m K"n D([!JAʼmLw@hR\* J+)pz+zۘ1KuԚB[ERPS6)+q\1/ar^P2$?'xbg/9nh5P @KQ ##|­% 'tJvb x {#1IU`qS ƜG (ˁ;HJ}4J-&!]?oIL?rtYd!eHxY[El4:Cp'$/#~D;1 hP @U`Oғ5s p-*8 j cp@8vؐ&guegxF;a0 %P״h,>  rnp@hy|3tϨ)<߇?b5+FQB (P`cMaq +t疗:O:o@h$g@'^Ij;zGͲVps=Vg42;iD>LcO/C&/l#mq[^A2 r~rpnADPxW~Q8vcʧrNC꽃␥=tSVp;0;%WC*,'*/VRKqRCj$9_hC:)8S ` Θ\ВQkq\U1Bèi8]twj 4-P @(RG?u$L(ysnB' LQ9Wu(hzPbmdu8G>*j2AAss= F@NGMͰ!7 J(d6>8~?yY.ukO_07V" :sɇP 6,˼hP @Q]!̟aeo4 miОzgby>IX @&4Ŏ-5Sk1T2 tq&;(ኍGi:D91=|^nA֫N)c+Ettz`Cg0'1sy Z㐳4ǤDl\1#_2xke^3n ?P˩Q@F%\P D|ggB=' ? >s90Os WyP @(L_po]Ww7C(8NWy^K0A  K+>"X,Nᵊ(]&VE9*TocD1r`nG6;G+?e6niA3 H$f^3oaZ&Nn0,-pKB(ǤW[wy~g];(qN"pxOUFq8dUv`au0ۻ4 M o:踵 8~t8?]z@xN%%,5);00NͿ @(+^)8_/h1u/48 6,@ @=w+B?Du古R+RY.I`VlBVe^:H!R* ӏv|!a\)|/CH <ҁGP Wc$|o%v}ەk_aGy8|pqe) ,QP @T^9sx]!pmlRRfָX  pYjt6#,R:^# `vJ8&@ۙ.4PO(hpN{ezcC]6Gz]vDrb=P{“;.f((WoלiIV mR >nRr!HЇ$=jJUܻ\Qzlz.}o )ڱ52c6 PJk9 cjZ̬ˮWPHfF±F @((0a79|oKy2qyN!fxk+ -,P}iL <&x:\ m`T2ӢJ.ɵ Qhz=:/?y(MY͖:XB^8v?v_0.%[@(8f5/36;0߫RM"}gfX8-Τвx+{kSuFMNh Ђ &W6HV^^ ahkҦ{Sx,tçF2P @((p HiJ0pKq Ng}e;HWt\LI[,Gd-[܅`)xk;:lNKeB,pOxxzOxB$(i=AP @()7 ; :÷!m5L<0)`@ jbɪeS0.ٝsn-4\yCvӳ !n~m[ubz@(x$;X]&Ne=FA>ɘ ^K~ sxW{#q7z PJo^~3[>A-,wp`;Xv1 wJ>p[a S8 M1|bB闡2TFP @(@x/Ns a7IBb/IMާb?xa:>sºVdRAwȱŇsS!>\ wxp4('e)sP׎c=sc;<g7sG[xmwE>5;8j<ܿ SZx(|IfR@xYJ(P`#?>?C_YRBVSjCFFyg8G#2A1f&cFŷC?Px #tt @(>)o>i!0etZR0!x;X͙NaYӳ9x\^SBᯄGV܃KX.Px/}o>O`bpw0;x?j8 z8|+Dp? P @( /'4}W/{ 8S,wp,-&#'U}o)a {çh* PyQ8P @(pQ o~ݷ^+oOۧ/ wȅ$K4C<hWseWwPfΖ|3C 'G Gm-v T~TP lCǾw]Sxc U{FZ}jx{>5 jg7E[;HQZP_d&,7plU;7g3loXEd DR+m(JGP @(ė}37NS;a֣饙o>;hbt u`\*^M'u1׎w+oY_OOXќW@A(gW`y~s~ #include "compiled_grammar_impl.h" #include "support/json_serializer.h" #include "testing.h" #include "tokenizer_info_impl.h" #include "xgrammar/exception.h" namespace xgrammar { /******************* AdaptiveTokenMask *******************/ AdaptiveTokenMask::AdaptiveTokenMask( size_t vocab_size, const std::vector>& sorted_decoded_vocab, const std::vector& accepted_indices, const std::vector& rejected_indices, const std::vector& uncertain_indices ) { auto size_acc = accepted_indices.size(); auto size_rej = rejected_indices.size(); store_type = size_acc >= USE_BITSET_THRESHOLD && size_rej >= USE_BITSET_THRESHOLD ? StoreType::kAcceptedBitset : size_acc < size_rej ? StoreType::kAccepted : StoreType::kRejected; if (store_type == StoreType::kAcceptedBitset) { accepted_bitset = DynamicBitset(vocab_size); for (auto idx : accepted_indices) { accepted_bitset.Set(sorted_decoded_vocab[idx].first, true); } } else if (store_type == StoreType::kAccepted) { this->accepted_indices = accepted_indices; } else { this->rejected_indices = rejected_indices; } this->uncertain_indices = uncertain_indices; } AdaptiveTokenMask::AdaptiveTokenMask( size_t vocab_size, const std::vector>& sorted_decoded_vocab, const std::vector& accepted_indices, const std::vector& uncertain_indices ) { auto size_acc = accepted_indices.size(); store_type = size_acc >= USE_BITSET_THRESHOLD ? StoreType::kAcceptedBitset : StoreType::kAccepted; if (store_type == StoreType::kAcceptedBitset) { accepted_bitset = DynamicBitset(vocab_size); for (auto idx : accepted_indices) { accepted_bitset.Set(sorted_decoded_vocab[idx].first, true); } } else { XGRAMMAR_DCHECK(store_type == StoreType::kAccepted); this->accepted_indices = accepted_indices; } this->uncertain_indices = uncertain_indices; } std::string AdaptiveTokenMask::Print(const TokenizerInfo& tokenizer_info) const { constexpr int kMaxPrintTokens = 100; std::stringstream ss; const auto& sorted_decoded_vocab = tokenizer_info.GetSortedDecodedVocab(); std::vector accepted_indices; std::vector rejected_indices; std::unordered_set uncertain_indices_set( uncertain_indices.begin(), uncertain_indices.end() ); accepted_indices.reserve(sorted_decoded_vocab.size()); rejected_indices.reserve(sorted_decoded_vocab.size()); if (store_type == StoreType::kAcceptedBitset) { for (int i = 0; i < static_cast(sorted_decoded_vocab.size()); ++i) { if (uncertain_indices_set.count(i)) { continue; } if (accepted_bitset[sorted_decoded_vocab[i].first]) { accepted_indices.push_back(i); } else { rejected_indices.push_back(i); } } } else if (store_type == StoreType::kAccepted) { accepted_indices = this->accepted_indices; // Reject indices = [0, sorted_decoded_vocab.size()) \ accepted_indices \ uncertain_indices int acc_ptr = 0; for (int i = 0; i < static_cast(sorted_decoded_vocab.size()); ++i) { while (acc_ptr < static_cast(accepted_indices.size()) && accepted_indices[acc_ptr] < i) { ++acc_ptr; } if (acc_ptr < static_cast(accepted_indices.size()) && accepted_indices[acc_ptr] == i) { continue; } if (uncertain_indices_set.count(i)) { continue; } rejected_indices.push_back(i); } } else { XGRAMMAR_DCHECK(store_type == StoreType::kRejected); rejected_indices = this->rejected_indices; // Accepted indices = [0, sorted_decoded_vocab.size()) \ rejected_indices \ uncertain_indices int rej_ptr = 0; for (int i = 0; i < static_cast(sorted_decoded_vocab.size()); ++i) { while (rej_ptr < static_cast(rejected_indices.size()) && rejected_indices[rej_ptr] < i) { ++rej_ptr; } if (rej_ptr < static_cast(rejected_indices.size()) && rejected_indices[rej_ptr] == i) { continue; } if (uncertain_indices_set.count(i)) { continue; } accepted_indices.push_back(i); } } std::string storage_type_str = store_type == StoreType::kAcceptedBitset ? "AcceptedBitset" : store_type == StoreType::kAccepted ? "Accepted" : "Rejected"; ss << "AdaptiveTokenMask(num_tokens=" << sorted_decoded_vocab.size() << ", accepted_num=" << accepted_indices.size() << ", rejected_num=" << rejected_indices.size() << ", uncertain_num=" << uncertain_indices.size() << ", storage_type=" << storage_type_str << ",\n"; // Convert indices to token ids for printing std::vector accepted_token_ids; std::vector rejected_token_ids; std::vector uncertain_token_ids; accepted_token_ids.reserve(accepted_indices.size()); rejected_token_ids.reserve(rejected_indices.size()); uncertain_token_ids.reserve(uncertain_indices.size()); for (auto idx : accepted_indices) { accepted_token_ids.push_back(sorted_decoded_vocab[idx].first); } std::sort(accepted_token_ids.begin(), accepted_token_ids.end()); for (auto idx : rejected_indices) { rejected_token_ids.push_back(sorted_decoded_vocab[idx].first); } std::sort(rejected_token_ids.begin(), rejected_token_ids.end()); for (auto idx : uncertain_indices) { uncertain_token_ids.push_back(sorted_decoded_vocab[idx].first); } std::sort(uncertain_token_ids.begin(), uncertain_token_ids.end()); ss << "accepted=" << PrintTokenByIds(accepted_token_ids, tokenizer_info, kMaxPrintTokens) << ",\nrejected=" << PrintTokenByIds(rejected_token_ids, tokenizer_info, kMaxPrintTokens) << ",\nuncertain=" << PrintTokenByIds(uncertain_token_ids, tokenizer_info, kMaxPrintTokens) << "\n)"; return ss.str(); } /************** CompiledGrammar::Impl **************/ picojson::value SerializeJSONValue(const CompiledGrammar::Impl& impl) { auto result = picojson::object{}; result["grammar"] = AutoSerializeJSONValue(impl.grammar); result["tokenizer_metadata"] = impl.tokenizer_info->DumpMetadataValue(); result["adaptive_token_mask_cache"] = AutoSerializeJSONValue(impl.adaptive_token_mask_cache); return picojson::value(result); } std::optional DeserializeJSONValue( CompiledGrammar::Impl* impl, const picojson::value& json_value, const TokenizerInfo& tokenizer_info ) { const auto& type_name = "CompiledGrammar"; if (!json_value.is()) { return ConstructDeserializeError("Expect an object", type_name); } const auto& object = json_value.get(); if (object.find("grammar") == object.end()) { return ConstructDeserializeError("Expect a 'grammar' field", type_name); } AutoDeserializeJSONValue(&(impl->grammar), object["grammar"], type_name); if (object.find("tokenizer_metadata") == object.end()) { return ConstructDeserializeError("Expect a 'tokenizer_metadata' field", type_name); } const auto& tokenizer_metadata = object["tokenizer_metadata"]; if (auto error = tokenizer_info->CheckMetadataMatch(tokenizer_metadata)) { return ConstructDeserializeError( std::string("Tokenizer metadata mismatch: ") + error->what(), type_name ); } impl->tokenizer_info = tokenizer_info; if (object.find("adaptive_token_mask_cache") == object.end()) { return ConstructDeserializeError("Expect a 'adaptive_token_mask_cache' field", type_name); } AutoDeserializeJSONValue(&(impl->adaptive_token_mask_cache), object["adaptive_token_mask_cache"]); return std::nullopt; } /************** CompiledGrammar **************/ std::size_t MemorySize(const CompiledGrammar::Impl& impl) { return MemorySize(impl.grammar) + MemorySize(impl.adaptive_token_mask_cache); } std::size_t CompiledGrammar::MemorySizeBytes() const { return MemorySize(*pimpl_); } Grammar CompiledGrammar::GetGrammar() const { return pimpl_->GetGrammar(); } TokenizerInfo CompiledGrammar::GetTokenizerInfo() const { return pimpl_->GetTokenizerInfo(); } /*! \brief Return the serialized JSON string of the compiled grammar. */ std::string CompiledGrammar::SerializeJSON() const { return AutoSerializeJSON(*this, true); } /*! \brief Deserialize a compiled grammar from a JSON string and tokenizer info. */ std::variant CompiledGrammar::DeserializeJSON( const std::string& json_string, const TokenizerInfo& tokenizer_info ) { picojson::value json_value; if (auto error = picojson::parse(json_value, json_string); !error.empty()) { return InvalidJSONError("Failed to parse JSON: " + error); } if (!json_value.is()) { return DeserializeFormatError("Expect an object"); } const auto& object = json_value.get(); if (auto error = SerializeVersion::Check(object)) { return error.value(); } auto impl = std::make_shared(); if (auto error = DeserializeJSONValue(impl.get(), json_value, tokenizer_info)) { return error.value(); } return CompiledGrammar(std::move(impl)); } } // namespace xgrammar xgrammar-0.2.3/cpp/compiled_grammar_impl.h000066400000000000000000000115541521764210300206170ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/compiled_grammar_impl.h * \brief The header for the data structures of the compiled grammar. */ #ifndef XGRAMMAR_COMPILED_GRAMMAR_IMPL_H_ #define XGRAMMAR_COMPILED_GRAMMAR_IMPL_H_ #include #include #include #include #include #include #include #include "earley_parser.h" #include "support/dynamic_bitset.h" #include "support/reflection.h" #include "xgrammar/compiler.h" #include "xgrammar/exception.h" namespace xgrammar { /******************* CompiledGrammar Datastructures *******************/ /*! * \brief Preprocessed information, for a given specific ParserState, divides the token set * into three categories: accepted, rejected, and uncertain. * Accepted: tokens that can be determined by the current ParserState to be acceptable * Rejected: tokens that can be determined by the current ParserState to be unacceptable * Uncertain: tokens that need the state of the parent ParserStates to determine if acceptable * * \note uncertain indices are stored directly. Accepted / rejected indices have three ways to * store to reduce memory and computation usage. See StoreType. * \note These indices are the indices of sorted_decoded_vocab in the CompiledGrammar * object, instead of the token ids. That helps the matching process. */ struct AdaptiveTokenMask { enum class StoreType { // Only store all accepted token indices. Then rejected indices = all_indices - accepted_indices // - uncertain_indices. This is useful when |accepted_indices| < |rejected_indices|. kAccepted = 0, // Only store all rejected token indices. Then accepted indices = all_indices - rejected_indices // - uncertain_indices. This is useful when |accepted_indices| > |rejected_indices|. kRejected = 1, // Store all accepted token indices in a bitset. This is useful when both |accepted_indices| and // |rejected_indices| are large. kAcceptedBitset = 2 }; StoreType store_type; static constexpr int USE_BITSET_THRESHOLD = 1000; std::vector accepted_indices; std::vector rejected_indices; DynamicBitset accepted_bitset; std::vector uncertain_indices; /*! \brief Default constructor. Only for deserialization. */ AdaptiveTokenMask() = default; AdaptiveTokenMask( size_t vocab_size, const std::vector>& sorted_decoded_vocab, const std::vector& accepted_indices, const std::vector& rejected_indices, const std::vector& uncertain_indices ); AdaptiveTokenMask( size_t vocab_size, const std::vector>& sorted_decoded_vocab, const std::vector& accepted_indices, const std::vector& uncertain_indices ); std::string Print(const TokenizerInfo& tokenizer_info) const; friend std::size_t MemorySize(const AdaptiveTokenMask& mask) { return MemorySize(mask.uncertain_indices) + MemorySize(mask.accepted_indices) + MemorySize(mask.rejected_indices) + MemorySize(mask.accepted_bitset); } }; XGRAMMAR_MEMBER_TABLE( AdaptiveTokenMask, "store_type", &AdaptiveTokenMask::store_type, "accepted_indices", &AdaptiveTokenMask::accepted_indices, "rejected_indices", &AdaptiveTokenMask::rejected_indices, "accepted_bitset", &AdaptiveTokenMask::accepted_bitset, "uncertain_indices", &AdaptiveTokenMask::uncertain_indices ); /*! * \brief All information that we need to match tokens in the tokenizer to the specified grammar. * It is the result of preprocessing. * \sa xgrammar::GrammarMatcher */ class CompiledGrammar::Impl { public: /*! \brief The grammar for the GrammarMatcher. */ Grammar grammar{NullObj{}}; /*! \brief The tokenizer information. */ TokenizerInfo tokenizer_info{NullObj{}}; /*! \brief Default constructor. */ Impl() = default; /*! \brief Mapping from the parser state to the adaptive token mask. */ std::unordered_map adaptive_token_mask_cache; Grammar GetGrammar() const { return grammar; } TokenizerInfo GetTokenizerInfo() const { return tokenizer_info; } friend struct member_trait; friend picojson::value SerializeJSONValue(const Impl& impl); friend std::optional DeserializeJSONValue( CompiledGrammar::Impl* impl, const picojson::value& json_value, const TokenizerInfo& tokenizer_info ); friend std::size_t MemorySize(const Impl& impl); }; XGRAMMAR_MEMBER_TABLE( CompiledGrammar::Impl, "grammar", &CompiledGrammar::Impl::grammar, "tokenizer_info", &CompiledGrammar::Impl::tokenizer_info, "adaptive_token_mask_cache", &CompiledGrammar::Impl::adaptive_token_mask_cache ); } // namespace xgrammar #endif // XGRAMMAR_COMPILED_GRAMMAR_IMPL_H_ xgrammar-0.2.3/cpp/config.cc000066400000000000000000000010071521764210300156670ustar00rootroot00000000000000/*! * Copyright (c) 2025 by Contributors * \file xgrammar/config.cc */ #include #include "support/json_serializer.h" #include "support/recursion_guard.h" namespace xgrammar { void SetMaxRecursionDepth(int max_recursion_depth) { RecursionGuard::SetMaxRecursionDepth(max_recursion_depth); } int GetMaxRecursionDepth() { return RecursionGuard::GetMaxRecursionDepth(); } std::string GetSerializationVersion() { return std::string(SerializeVersion::GetVersion()); } } // namespace xgrammar xgrammar-0.2.3/cpp/earley_parser.cc000066400000000000000000001076401521764210300172710ustar00rootroot00000000000000/*! * Copyright (c) 2025 by Contributors * \file xgrammar/earley_parser.cc */ #include "earley_parser.h" #include #include #include #include #include #include #include #include "fsm.h" #include "grammar_impl.h" #include "support/encoding.h" #include "support/logging.h" #include "xgrammar/grammar.h" namespace xgrammar { using GrammarExprType = Grammar::Impl::GrammarExprType; using GrammarExpr = Grammar::Impl::GrammarExpr; bool EarleyParser::IsCompleted() const { return is_completed_.back(); } void EarleyParser::PopLastStates(int32_t cnt) { if (stop_token_is_accepted_) { stop_token_is_accepted_ = false; } if (cnt >= static_cast(rule_id_to_completable_states_.size())) { XGRAMMAR_LOG(FATAL) << "The number of states to be popped is larger than the size of states."; } rule_id_to_completable_states_.PopBack(cnt); is_completed_.erase(is_completed_.end() - cnt, is_completed_.end()); scanable_state_history_.PopBack(cnt); } void EarleyParser::Complete(const ParserState& state, bool debug_print) { // Check if a rule is completed. if (state.rule_start_pos == ParserState::kNoPrevInputPos) { // assert: if a root rule can achieve here, then it must be completed. if (debug_print) { XGRAMMAR_LOG(INFO) << "The root rule is completed."; } tmp_accept_stop_token_ = true; return; } if (debug_print) { XGRAMMAR_LOG(INFO) << "The rule " << state.rule_id << ": " << grammar_->GetRule(state.rule_id).name << " is completed, trying to complete its parent states."; } // Check all the possible parent states. const auto& parent_states_map = rule_id_to_completable_states_[state.rule_start_pos]; for (const auto& [ref_id, parent_state] : parent_states_map) { if (ref_id != state.rule_id) { continue; } XGRAMMAR_DCHECK( parent_state.rule_id == -1 || grammar_->per_rule_fsms[parent_state.rule_id].has_value() ); if (parent_state.rule_id == -1) { const auto& parent_expr = grammar_->GetGrammarExpr(parent_state.sequence_id); const auto& element_expr = grammar_->GetGrammarExpr(parent_expr[parent_state.element_id]); // The new rule is not referenced by a fsm. XGRAMMAR_DCHECK( element_expr.type == GrammarExprType::kRuleRef || element_expr.type == GrammarExprType::kRepeat ); if (element_expr.type == GrammarExprType::kRuleRef) { Enqueue(ParserState{ parent_state.rule_id, parent_state.sequence_id, parent_state.element_id + 1, parent_state.rule_start_pos, 0 }); continue; } XGRAMMAR_DCHECK(element_expr.type == GrammarExprType::kRepeat); // The parent state is a repeat, we need to increase the repeat count. auto new_state = parent_state; const int32_t& min_repeat_count = element_expr[1]; const int32_t& max_repeat_count = element_expr[2]; new_state.repeat_count++; // The repeat rule can be completed, and we advance the state. Don't forget to // reset the repeat count. if (new_state.repeat_count >= min_repeat_count) { Enqueue(ParserState{ parent_state.rule_id, parent_state.sequence_id, parent_state.element_id + 1, parent_state.rule_start_pos, 0 }); } // If the repeat count is less than the max repeat count, we can continue to // visit the repeat state for another round. if (new_state.repeat_count < max_repeat_count) { Enqueue(new_state); } continue; } // If the rule is referenced by a fsm, we need to advance the fsm. XGRAMMAR_DCHECK(grammar_->per_rule_fsms[parent_state.rule_id].has_value()); // Check if the parent_state sits on a kRepeatRef edge bool handled_as_repeat = false; const auto& parent_fsm = grammar_->per_rule_fsms[parent_state.rule_id].value(); for (const auto& edge : parent_fsm.GetFsm().GetFsm().GetEdges(parent_state.element_id)) { // Because of invariance, a state with a kRepeatRef edge has exactly one outgoing edge. if (!edge.IsRepeatRef()) continue; auto info = grammar_->complete_fsm.GetRepeatEdgeInfo(edge.GetAuxIndex()); if (info.RuleId() != ref_id) continue; handled_as_repeat = true; int32_t new_count = parent_state.repeat_count + 1; if (new_count >= info.Lower()) { Enqueue(ParserState{ parent_state.rule_id, parent_state.sequence_id, edge.target, parent_state.rule_start_pos, 0, 0 }); } if (new_count < info.Upper()) { Enqueue(ParserState{ parent_state.rule_id, parent_state.sequence_id, parent_state.element_id, parent_state.rule_start_pos, 0, new_count }); } break; } if (!handled_as_repeat) { Enqueue(parent_state); } } } std::pair EarleyParser::Predict( const ParserState& state, bool debug_print ) { // Check if the rule has a corresponding FSM. if (state.rule_id != -1) { XGRAMMAR_DCHECK(grammar_->per_rule_fsms[state.rule_id].has_value()); // Try to expand the fsm. ExpandNextRuleRefElementOnFSM(state, debug_print); const auto& fsm = grammar_->per_rule_fsms[state.rule_id].value(); return std::make_pair( fsm.GetFsm().IsScanableState(state.element_id), fsm.GetFsm().IsEndState(state.element_id) ); } const GrammarExpr& grammar_expr = grammar_->GetGrammarExpr(state.sequence_id); XGRAMMAR_DCHECK( grammar_expr.type == GrammarExprType::kSequence || grammar_expr.type == GrammarExprType::kEmptyStr ); if (state.element_id == grammar_expr.size()) { // The rule is completed. return std::make_pair(false, true); } const auto& element_expr = grammar_->GetGrammarExpr(grammar_expr[state.element_id]); switch (element_expr.type) { case GrammarExprType::kRuleRef: { ExpandNextRuleRefElement(state, grammar_expr, &element_expr, debug_print); return std::make_pair(false, false); } case GrammarExprType::kCharacterClassStar: { if (state.sub_element_id == 0) { Enqueue(ParserState{ state.rule_id, state.sequence_id, state.element_id + 1, state.rule_start_pos, 0 }); } return std::make_pair(true, false); } case GrammarExprType::kRepeat: { const int32_t& min_repeat_count = element_expr[1]; const int32_t& max_repeat_count = element_expr[2]; // If the current repeat count is less than the max repeat count, // we can expand the next rule reference element. XGRAMMAR_DCHECK(state.repeat_count <= max_repeat_count); ExpandNextRuleRefElement(state, grammar_expr, &element_expr, debug_print); if (state.repeat_count >= min_repeat_count) { Enqueue(ParserState{ state.rule_id, state.sequence_id, state.element_id + 1, state.rule_start_pos, 0 }); } return std::make_pair(false, false); } case GrammarExprType::kByteString: case GrammarExprType::kCharacterClass: { return std::make_pair(true, false); // The element is scanable, but not completable. } case GrammarExprType::kToken: case GrammarExprType::kExcludeToken: { return std::make_pair(false, false); } default: { XGRAMMAR_LOG(FATAL) << "The element type is not supported! The type is: " << int(element_expr.type); XGRAMMAR_UNREACHABLE(); } } } void EarleyParser::Scan(const ParserState& state, const uint8_t ch) { XGRAMMAR_DCHECK(state.rule_id == -1 || grammar_->per_rule_fsms[state.rule_id].has_value()); if (state.rule_id == -1) { const auto& cur_rule = grammar_->GetGrammarExpr(state.sequence_id); const auto& element_expr = grammar_->GetGrammarExpr(cur_rule[state.element_id]); // The element is a rule reference, we do not need to scan it. switch (element_expr.type) { case (GrammarExprType::kByteString): { AdvanceByteString(state, ch, element_expr); break; } case (GrammarExprType::kCharacterClass): { AdvanceCharacterClass(state, ch, element_expr); break; } case (GrammarExprType::kCharacterClassStar): { AdvanceCharacterClassStar(state, ch, element_expr); break; } default: { XGRAMMAR_LOG(FATAL) << "The element type is not supported! The type is: " << int(element_expr.type); XGRAMMAR_UNREACHABLE(); } } } else { AdvanceFsm(state, ch); } } /*! \note The workflow of Advance is as follows: 1. Scan all the states in the latest states. Add all the possible states to the next states. 2. If the next states are empty, then the character is not accepted. 3. If the next states are not empty, then the character is accepted. Moreover, we need to complete and predict the next states. \note Thus, when initializing the Earley parser, we need to add the initial state to the history_states[0], and perform prediction and completion on the initial state. */ bool EarleyParser::Advance(const uint8_t ch, bool debug_print) { // Initialize the containers. XGRAMMAR_DCHECK(tmp_process_state_queue_.empty()) << "The tmp_process_state_queue_ should be empty before the scan."; tmp_states_visited_in_queue_.Clear(); tmp_states_to_be_added_.clear(); tmp_accept_stop_token_ = false; const auto& latest_states = scanable_state_history_[scanable_state_history_.size() - 1]; // Scan all the scanable states. for (const auto& state : latest_states) { Scan(state, ch); } // Check if the character is accepted. if (tmp_process_state_queue_.empty() && tmp_states_to_be_added_.empty()) { return false; } // execute Predict and Complete for all states in the queue until empty. rule_id_to_completable_states_.PushBack(std::vector>()); while (!tmp_process_state_queue_.empty()) { const auto state = std::move(tmp_process_state_queue_.front()); tmp_process_state_queue_.pop(); auto [scanable, completable] = Predict(state, debug_print); if (completable) { Complete(state, debug_print); } if (scanable) { tmp_states_to_be_added_.push_back(state); } } // Check if the grammar is completed, and add the scannable states to the history. is_completed_.push_back(tmp_accept_stop_token_); scanable_state_history_.PushBack(tmp_states_to_be_added_); return true; } EarleyParser::EarleyParser( const Grammar& grammar, const ParserState& init_state, const bool need_expand ) : grammar_(grammar) { if (!grammar->optimized) { XGRAMMAR_LOG(FATAL) << "The grammar is not optimized. Please optimize the grammar before using " "the Earley parser."; } // Check if the initial state is valid. If invalid, then we choose the root state as default. ParserState init = init_state; if (init_state.IsInvalid()) { init = ParserState( grammar_->GetRootRuleId(), ParserState::kUnexpandedRuleStartSequenceId, 0, ParserState::kNoPrevInputPos, 0 ); } else { init = init_state; } // If there is no need to expand the initial state, we only need to add it to the // scanable states history. if (!need_expand) { rule_id_to_completable_states_.PushBack(std::vector>()); is_completed_.push_back(false); scanable_state_history_.PushBack({init}); return; } // Otherwise, we expand the initial state, and process the queue. PushStateAndExpand(init); } void EarleyParser::PushStateAndExpand(const ParserState& state) { tmp_states_visited_in_queue_.Clear(); tmp_accept_stop_token_ = false; tmp_states_to_be_added_.clear(); // If the rule can't be expanded, we need to add it to the queue. if (!ExpandAndEnqueueUnexpandedState(state)) { Enqueue(state); } rule_id_to_completable_states_.PushBack(std::vector>()); while (!tmp_process_state_queue_.empty()) { const auto state = tmp_process_state_queue_.front(); tmp_process_state_queue_.pop(); auto [scanable, completable] = Predict(state); if (completable) { Complete(state); } if (scanable) { tmp_states_to_be_added_.push_back(state); } } is_completed_.push_back(tmp_accept_stop_token_); scanable_state_history_.PushBack(tmp_states_to_be_added_); } void EarleyParser::Reset() { rule_id_to_completable_states_.PopBack(rule_id_to_completable_states_.size()); scanable_state_history_.PopBack(scanable_state_history_.size()); is_completed_.clear(); stop_token_is_accepted_ = false; XGRAMMAR_DCHECK(tmp_process_state_queue_.empty()); PushStateAndExpand(ParserState( grammar_->GetRootRuleId(), ParserState::kUnexpandedRuleStartSequenceId, 0, ParserState::kNoPrevInputPos, 0 )); } bool EarleyParser::ExpandAndEnqueueUnexpandedState(const ParserState& state) { if (state.sequence_id != ParserState::kUnexpandedRuleStartSequenceId) { return false; } auto cur_rule_id = state.rule_id; auto cur_rule_body_id = grammar_->GetRule(cur_rule_id).body_expr_id; XGRAMMAR_DCHECK(state.rule_id != -1 && grammar_->per_rule_fsms[state.rule_id].has_value()); Enqueue(ParserState{ cur_rule_id, cur_rule_body_id, grammar_->per_rule_fsms[state.rule_id]->GetFsm().GetStart(), ParserState::kNoPrevInputPos, 0 }); return true; } void EarleyParser::ExpandNextRuleRefElement( const ParserState& state, const GrammarExpr& grammar_expr, const GrammarExpr* sub_grammar_expr, bool debug_print ) { // Path A. The rule has a corresponding FSM. XGRAMMAR_DCHECK(!(state.rule_id != -1 && grammar_->per_rule_fsms[state.rule_id].has_value())); XGRAMMAR_DCHECK(grammar_expr.type == GrammarExprType::kSequence); XGRAMMAR_DCHECK( sub_grammar_expr->type == GrammarExprType::kRuleRef || sub_grammar_expr->type == GrammarExprType::kRepeat ); auto ref_rule_id = (*sub_grammar_expr)[0]; if (debug_print) { XGRAMMAR_LOG(INFO) << "The rule " << state.rule_id << ": " << grammar_->GetRule(state.rule_id).name << " predict the new rule " << ref_rule_id << ": " << grammar_->GetRule(ref_rule_id).name << "."; } bool right_recursion_to_root = false; if (state.element_id != grammar_expr.size() - 1 || sub_grammar_expr->type == GrammarExprType::kRepeat || (state.rule_start_pos == rule_id_to_completable_states_.size() - 1)) { // It's not the right recursion, or it's the root rule. rule_id_to_completable_states_.PushBackInLatestRow(std::make_pair(ref_rule_id, state)); } else { if (state.rule_start_pos == ParserState::kNoPrevInputPos) { right_recursion_to_root = true; } else { // If it's the right recursion, we need to add the ancestors of the parent state. const auto in_vec = [&](const ParserState& state_) { return std::find_if( rule_id_to_completable_states_.Back().begin(), rule_id_to_completable_states_.Back().end(), [&](const auto& s) { return StateEqualForParsing()(s.second, state_) && s.first == ref_rule_id; } ) != rule_id_to_completable_states_.Back().end(); }; const auto& parent_states_map = rule_id_to_completable_states_[state.rule_start_pos]; std::vector> to_added_states; for (const auto& parent_state_iter : parent_states_map) { if (parent_state_iter.first != state.rule_id) continue; const auto& parent_state = parent_state_iter.second; if (!in_vec(parent_state)) { to_added_states.push_back({ref_rule_id, parent_state}); } } for (const auto& to_add_state : to_added_states) { rule_id_to_completable_states_.PushBackInLatestRow(to_add_state); } } } if (std::find( grammar_->allow_empty_rule_ids.begin(), grammar_->allow_empty_rule_ids.end(), ref_rule_id ) != grammar_->allow_empty_rule_ids.end()) { XGRAMMAR_DCHECK(grammar_expr.type == GrammarExprType::kSequence); Enqueue( ParserState{state.rule_id, state.sequence_id, state.element_id + 1, state.rule_start_pos, 0} ); } // If the reference rule is not visited, we need to add it to the queue. const auto& ref_rule = grammar_->GetRule(ref_rule_id); const auto& ref_grammar_expr_id = ref_rule.body_expr_id; XGRAMMAR_DCHECK(grammar_->per_rule_fsms[ref_rule_id].has_value()); if (std::find( grammar_->allow_empty_rule_ids.begin(), grammar_->allow_empty_rule_ids.end(), ref_rule_id ) != grammar_->allow_empty_rule_ids.end()) { Enqueue( ParserState{state.rule_id, state.sequence_id, state.element_id + 1, state.rule_start_pos, 0} ); } const auto& ref_fsm = grammar_->per_rule_fsms[ref_rule_id].value(); Enqueue(ParserState{ ref_rule_id, ref_grammar_expr_id, ref_fsm.GetFsm().GetStart(), right_recursion_to_root ? ParserState::kNoPrevInputPos : int32_t(rule_id_to_completable_states_.size() - 1), 0 }); } void EarleyParser::ExpandNextRuleRefElementOnFSM(const ParserState& state, bool debug_print) { XGRAMMAR_DCHECK(state.rule_id != -1 && grammar_->per_rule_fsms[state.rule_id].has_value()); const auto& fsm = grammar_->per_rule_fsms[state.rule_id].value(); // Add the rule reference pairs, and enqueue the epsilon edges. for (const auto& edge : fsm.GetFsm().GetFsm().GetEdges(state.element_id)) { if (edge.IsEpsilon()) { Enqueue(ParserState{state.rule_id, state.sequence_id, edge.target, state.rule_start_pos, 0}); continue; } int target; int ref_rule_id; bool is_repeat = false; RepeatEdgeRef repeat_info{nullptr}; if (edge.IsRuleRef()) { target = edge.target; ref_rule_id = edge.GetRefRuleId(); } else if (edge.IsRepeatRef()) { is_repeat = true; repeat_info = grammar_->complete_fsm.GetRepeatEdgeInfo(edge.GetAuxIndex()); target = edge.target; ref_rule_id = repeat_info.RuleId(); if (state.repeat_count >= repeat_info.Lower()) { Enqueue(ParserState{state.rule_id, state.sequence_id, target, state.rule_start_pos, 0, 0}); } if (state.repeat_count >= repeat_info.Upper()) { continue; } } else { continue; } bool right_recursion_to_root = false; if (debug_print) { XGRAMMAR_LOG(INFO) << "The rule " << state.rule_id << ": " << grammar_->GetRule(state.rule_id).name << " predict the new rule " << ref_rule_id << ": " << grammar_->GetRule(ref_rule_id).name << "."; } if (!is_repeat && (fsm.GetFsm().GetFsm().GetEdges(target).size() == 0) && fsm.GetFsm().IsEndState(target) && state.rule_start_pos != static_cast(rule_id_to_completable_states_.size() - 1)) { // It's a right recursion. We can optimize it. // If it's the right recursion, we need to add the ancestors of the parent state. if (state.rule_start_pos == ParserState::kNoPrevInputPos) { // In this case, we can mark the new state as the root state to speed up. right_recursion_to_root = true; } else { const auto in_vec = [&](const ParserState& state_) { return std::find_if( rule_id_to_completable_states_.Back().begin(), rule_id_to_completable_states_.Back().end(), [&](const auto& s) { return StateEqualForParsing()(s.second, state_) && s.first == ref_rule_id; } ) != rule_id_to_completable_states_.Back().end(); }; const auto& parent_states_map = rule_id_to_completable_states_[state.rule_start_pos]; std::vector> to_added_states; for (const auto& parent_state_iter : parent_states_map) { if (parent_state_iter.first != state.rule_id) continue; const auto& parent_state = parent_state_iter.second; if (!in_vec(parent_state)) { to_added_states.push_back({ref_rule_id, parent_state}); } } for (const auto& to_add_state : to_added_states) { rule_id_to_completable_states_.PushBackInLatestRow(to_add_state); } } } else { if (is_repeat) { // For kRepeatRef: store element_id = source state, preserve repeat_count rule_id_to_completable_states_.PushBackInLatestRow( {ref_rule_id, ParserState{ state.rule_id, state.sequence_id, state.element_id, state.rule_start_pos, 0, state.repeat_count }} ); } else { // For kRuleRef: store element_id = target (post-transition state) rule_id_to_completable_states_.PushBackInLatestRow( {ref_rule_id, ParserState{state.rule_id, state.sequence_id, target, state.rule_start_pos, 0}} ); } } // Check if the reference rule can be empty. if (!is_repeat && std::binary_search( grammar_->allow_empty_rule_ids.begin(), grammar_->allow_empty_rule_ids.end(), ref_rule_id )) { Enqueue(ParserState{state.rule_id, state.sequence_id, target, state.rule_start_pos, 0}); } // If the reference rule is not visited, we need to add it to the queue. const auto& ref_rule = grammar_->GetRule(ref_rule_id); const auto& ref_grammar_expr_id = ref_rule.body_expr_id; XGRAMMAR_DCHECK(grammar_->per_rule_fsms[ref_rule_id].has_value()); if (!is_repeat && std::binary_search( grammar_->allow_empty_rule_ids.begin(), grammar_->allow_empty_rule_ids.end(), ref_rule_id )) { Enqueue(ParserState{state.rule_id, state.sequence_id, target, state.rule_start_pos, 0}); } const auto& ref_fsm = grammar_->per_rule_fsms[ref_rule_id].value(); Enqueue(ParserState{ ref_rule_id, ref_grammar_expr_id, ref_fsm.GetFsm().GetStart(), right_recursion_to_root ? ParserState::kNoPrevInputPos : int32_t(rule_id_to_completable_states_.size() - 1), 0 }); } } void EarleyParser::AdvanceByteString( const ParserState& state, const uint8_t ch, const GrammarExpr& sub_rule ) { XGRAMMAR_DCHECK(sub_rule.type == GrammarExprType::kByteString); XGRAMMAR_DCHECK(sub_rule.size() > state.sub_element_id); if (static_cast(sub_rule[state.sub_element_id]) == ch) { auto new_state = state; new_state.sub_element_id++; if (new_state.sub_element_id == sub_rule.size()) { new_state.element_id++; new_state.sub_element_id = 0; Enqueue(new_state); // Assert: In a sequence, the bytestring can't be skipped. So the state can't be repeated. } else { tmp_states_to_be_added_.push_back(new_state); } } return; } void EarleyParser::AdvanceCharacterClass( const ParserState& state, const uint8_t ch, const GrammarExpr& sub_sequence ) { XGRAMMAR_DCHECK(sub_sequence.type == GrammarExprType::kCharacterClass) << "The element type is not supported!"; bool is_negative = static_cast(sub_sequence[0]); // The state is matching a UTF8 character (continuation bytes). if (state.sub_element_id > 0) { if ((ch & 0xC0) == 0x80) { auto new_state = state; new_state.sub_element_id--; // Accumulate the codepoint from continuation byte new_state.partial_codepoint = (new_state.partial_codepoint << 6) | (ch & 0x3F); // Check if the UTF8 character is completed. if (new_state.sub_element_id == 0) { if (is_negative) { // For negative classes, accept if codepoint is NOT in any range bool matches_range = false; for (int i = 1; i < sub_sequence.size(); i += 2) { if (new_state.partial_codepoint >= sub_sequence[i] && new_state.partial_codepoint <= sub_sequence[i + 1]) { matches_range = true; break; } } if (!matches_range) { new_state.element_id++; new_state.partial_codepoint = 0; Enqueue(new_state); } } else { // For positive classes, accept if codepoint IS in a range bool matches_range = false; for (int i = 1; i < sub_sequence.size(); i += 2) { if (new_state.partial_codepoint >= sub_sequence[i] && new_state.partial_codepoint <= sub_sequence[i + 1]) { matches_range = true; break; } } if (matches_range) { new_state.element_id++; new_state.partial_codepoint = 0; Enqueue(new_state); } } } else { // Check if partial codepoint could still potentially match any range int32_t remaining_bytes = new_state.sub_element_id; int32_t min_codepoint = new_state.partial_codepoint << (6 * remaining_bytes); int32_t max_codepoint = min_codepoint | ((1 << (6 * remaining_bytes)) - 1); bool could_match = false; for (int i = 1; i < sub_sequence.size(); i += 2) { int32_t lower = sub_sequence[i]; int32_t upper = sub_sequence[i + 1]; if (max_codepoint >= lower && min_codepoint <= upper) { could_match = true; break; } } // For negative classes: always continue (will verify on final byte) // For positive classes: only continue if some range could match bool should_continue = is_negative ? true : could_match; if (should_continue) { tmp_states_to_be_added_.push_back(new_state); } } } return; } // Handle non-ASCII first bytes if (!isascii(ch)) { auto [accepted, num_bytes, partial] = HandleUTF8FirstByte(ch); if (!accepted) { return; } XGRAMMAR_DCHECK(num_bytes > 1); // Compute possible codepoint range for this first byte int32_t min_codepoint = partial << (6 * (num_bytes - 1)); int32_t max_codepoint = min_codepoint | ((1 << (6 * (num_bytes - 1))) - 1); // Check if any stored range could potentially match bool could_match = false; for (int i = 1; i < sub_sequence.size(); i += 2) { int32_t lower = sub_sequence[i]; int32_t upper = sub_sequence[i + 1]; // Check for overlap between [min_codepoint, max_codepoint] and [lower, upper] if (max_codepoint >= lower && min_codepoint <= upper) { could_match = true; break; } } // For negative classes: accept if no range could match (will verify on final byte) // For positive classes: accept if some range could match (will verify on final byte) bool should_continue = is_negative ? true : could_match; if (should_continue) { auto new_state = state; new_state.sub_element_id = num_bytes - 1; new_state.partial_codepoint = partial; tmp_states_to_be_added_.push_back(new_state); } return; } // ASCII handling (unchanged) for (int i = 1; i < sub_sequence.size(); i += 2) { if (static_cast(sub_sequence[i]) <= ch && ch <= static_cast(sub_sequence[i + 1])) { if (!is_negative) { auto new_state = state; new_state.element_id++; new_state.sub_element_id = 0; Enqueue(new_state); } return; } } if (is_negative) { auto new_state = state; new_state.element_id++; new_state.sub_element_id = 0; Enqueue(new_state); } } void EarleyParser::AdvanceCharacterClassStar( const ParserState& state, const uint8_t ch, const GrammarExpr& sub_sequence ) { XGRAMMAR_DCHECK(sub_sequence.type == GrammarExprType::kCharacterClassStar) << "The element type is not supported!"; bool is_negative = static_cast(sub_sequence[0]); // The state is matching a UTF8 character (continuation bytes). if (state.sub_element_id > 0) { if ((ch & 0xC0) == 0x80) { auto new_state = state; new_state.sub_element_id--; // Accumulate the codepoint from continuation byte new_state.partial_codepoint = (new_state.partial_codepoint << 6) | (ch & 0x3F); // Check if the UTF8 character is completed. if (new_state.sub_element_id == 0) { if (is_negative) { // For negative classes, accept if codepoint is NOT in any range bool matches_range = false; for (int i = 1; i < sub_sequence.size(); i += 2) { if (new_state.partial_codepoint >= sub_sequence[i] && new_state.partial_codepoint <= sub_sequence[i + 1]) { matches_range = true; break; } } if (!matches_range) { new_state.partial_codepoint = 0; Enqueue(new_state); } } else { // For positive classes, accept if codepoint IS in a range bool matches_range = false; for (int i = 1; i < sub_sequence.size(); i += 2) { if (new_state.partial_codepoint >= sub_sequence[i] && new_state.partial_codepoint <= sub_sequence[i + 1]) { matches_range = true; break; } } if (matches_range) { new_state.partial_codepoint = 0; Enqueue(new_state); } } } else { // Check if partial codepoint could still potentially match any range int32_t remaining_bytes = new_state.sub_element_id; int32_t min_codepoint = new_state.partial_codepoint << (6 * remaining_bytes); int32_t max_codepoint = min_codepoint | ((1 << (6 * remaining_bytes)) - 1); bool could_match = false; for (int i = 1; i < sub_sequence.size(); i += 2) { int32_t lower = sub_sequence[i]; int32_t upper = sub_sequence[i + 1]; if (max_codepoint >= lower && min_codepoint <= upper) { could_match = true; break; } } // For negative classes: always continue (will verify on final byte) // For positive classes: only continue if some range could match bool should_continue = is_negative ? true : could_match; if (should_continue) { tmp_states_to_be_added_.push_back(new_state); } } } return; } // Handle non-ASCII first bytes if (!isascii(ch)) { auto [accepted, num_bytes, partial] = HandleUTF8FirstByte(ch); if (!accepted) { return; } XGRAMMAR_DCHECK(num_bytes > 1); // Compute possible codepoint range for this first byte int32_t min_codepoint = partial << (6 * (num_bytes - 1)); int32_t max_codepoint = min_codepoint | ((1 << (6 * (num_bytes - 1))) - 1); // Check if any stored range could potentially match bool could_match = false; for (int i = 1; i < sub_sequence.size(); i += 2) { int32_t lower = sub_sequence[i]; int32_t upper = sub_sequence[i + 1]; // Check for overlap between [min_codepoint, max_codepoint] and [lower, upper] if (max_codepoint >= lower && min_codepoint <= upper) { could_match = true; break; } } // For negative classes: accept if no range could match (will verify on final byte) // For positive classes: accept if some range could match (will verify on final byte) bool should_continue = is_negative ? true : could_match; if (should_continue) { auto new_state = state; new_state.sub_element_id = num_bytes - 1; new_state.partial_codepoint = partial; tmp_states_to_be_added_.push_back(new_state); } return; } // ASCII handling (unchanged) for (int i = 1; i < sub_sequence.size(); i += 2) { if (static_cast(sub_sequence[i]) <= ch && ch <= static_cast(sub_sequence[i + 1])) { if (!is_negative) { Enqueue(state); } return; } } if (is_negative) { Enqueue(state); } } void EarleyParser::AdvanceFsm(const ParserState& state, const uint8_t ch) { XGRAMMAR_DCHECK(state.rule_id != -1 && grammar_->per_rule_fsms[state.rule_id].has_value()); const auto& current_fsm = grammar_->per_rule_fsms[state.rule_id].value(); for (const auto& edge : current_fsm.GetFsm().GetFsm().GetEdges(state.element_id)) { if ((!edge.IsCharRange()) || ch < edge.min || ch > edge.max) { continue; } auto new_state = state; new_state.element_id = edge.target; if ((!current_fsm.GetFsm().IsNonTerminalState(edge.target)) && (!current_fsm.GetFsm().IsEndState(edge.target) && current_fsm.GetFsm().IsScanableState(edge.target))) { EnqueueWithoutProcessing(std::move(new_state)); } else { Enqueue(std::move(new_state)); } } } void EarleyParser::ScanAtomicToken(const ParserState& state, int32_t token_id) { if (state.rule_id == -1) return; XGRAMMAR_DCHECK(grammar_->per_rule_fsms[state.rule_id].has_value()); const auto& current_fsm = grammar_->per_rule_fsms[state.rule_id].value(); for (const auto& edge : current_fsm.GetFsm().GetFsm().GetEdges(state.element_id)) { bool matched = false; if (edge.IsToken()) { auto info = current_fsm.GetFsm().GetFsm().GetTokenEdgeInfo(edge.GetAuxIndex()); matched = info.Contains(token_id); } else if (edge.IsExcludeToken()) { auto info = current_fsm.GetFsm().GetFsm().GetExcludeTokenEdgeInfo(edge.GetAuxIndex()); matched = info.Accepts(token_id); } if (!matched) continue; auto new_state = state; new_state.element_id = edge.target; if ((!current_fsm.GetFsm().IsNonTerminalState(edge.target)) && (!current_fsm.GetFsm().IsEndState(edge.target) && current_fsm.GetFsm().IsScanableState(edge.target))) { EnqueueWithoutProcessing(std::move(new_state)); } else { Enqueue(std::move(new_state)); } } } bool EarleyParser::AdvanceAtomicToken(int32_t token_id, bool debug_print) { XGRAMMAR_DCHECK(tmp_process_state_queue_.empty()) << "The tmp_process_state_queue_ should be empty before AdvanceAtomicToken."; tmp_states_visited_in_queue_.Clear(); tmp_states_to_be_added_.clear(); tmp_accept_stop_token_ = false; const auto& latest_states = scanable_state_history_[scanable_state_history_.size() - 1]; for (const auto& state : latest_states) { ScanAtomicToken(state, token_id); } if (tmp_process_state_queue_.empty() && tmp_states_to_be_added_.empty()) { return false; } rule_id_to_completable_states_.PushBack(std::vector>()); while (!tmp_process_state_queue_.empty()) { const auto state = std::move(tmp_process_state_queue_.front()); tmp_process_state_queue_.pop(); auto [scanable, completable] = Predict(state, debug_print); if (completable) { Complete(state, debug_print); } if (scanable) { tmp_states_to_be_added_.push_back(state); } } is_completed_.push_back(tmp_accept_stop_token_); scanable_state_history_.PushBack(tmp_states_to_be_added_); return true; } bool RepeatDetector::IsVisited(const ParserState& state) const { // If the size is larger than the threshold, then we use the set to check. if (size_ > transition_threshold_) { return visited_set_.find(state) != visited_set_.end(); } return std::find_if( visited_vector_.begin(), visited_vector_.begin() + size_, [&state](const ParserState& s) { return StateEqualForParsing()(state, s); } ) != visited_vector_.begin() + size_; } void RepeatDetector::Insert(const ParserState& state) { if (size_ == transition_threshold_) { for (const auto& s : visited_vector_) { visited_set_.insert(s); } } size_++; if (size_ > transition_threshold_) { visited_set_.insert(state); } else { visited_vector_[size_ - 1] = state; } } void RepeatDetector::Clear() { if (size_ > transition_threshold_) { visited_set_.clear(); } size_ = 0; } } // namespace xgrammar xgrammar-0.2.3/cpp/earley_parser.h000066400000000000000000000430751521764210300171340ustar00rootroot00000000000000/*! * Copyright (c) 2025 by Contributors * \file xgrammar/earley_parser.h * \brief The header for the definition of the Earley parser. */ #ifndef XGRAMMAR_EARLEY_PARSER_H_ #define XGRAMMAR_EARLEY_PARSER_H_ #include #include #include #include #include #include #include "grammar_impl.h" #include "support/compact_2d_array.h" #include "support/utils.h" #include "xgrammar/grammar.h" namespace xgrammar { /*! * \brief The state of the Earley parser. * In the implementation, a rule can only be a kchoices or a ktagdispatch. * A kchoices rule must be composed of some ksequence rules, or a kemptyrule. * In the ksequence, every element in the sequence must be a kbytestring, a * kcharacterclass, a kcharacterclassstar, or a rule reference. * * - rule_id: The id of the rule. * - sequence_id: The id of the sequence in the rule. * - element_id: The id of the element in the sequence, or the id of the node in * the tag dispatch fsm. * - rule_start_pos: The id of the parent node in the Earley parser. i.e. the rule * is predicted from the k-th character. * - sub_element_id: The id of the sub element in the current element, i.e.: * - kbytestring: the id of the byte in the string. * - kcharacterclass: How many bytes are left to be read in the utf8 character. * - kcharacterclassstar: How many bytes are left to be read in the utf8 character. */ struct ParserState { constexpr ParserState() = default; constexpr ParserState( const int32_t& rule_id, const int32_t& sequence_id, const int32_t& element_id, const int32_t& rule_start_pos, const int32_t& sub_element_id, const int32_t& repeat_count = 0, const int32_t& partial_codepoint = 0 ) : rule_id(rule_id), sequence_id(sequence_id), element_id(element_id), rule_start_pos(rule_start_pos), sub_element_id(sub_element_id), repeat_count(repeat_count), partial_codepoint(partial_codepoint) {} /*! * \brief A sequence_id value of kUnexpandedRuleStartSequenceId means a rule hasn't been * expanded. */ static constexpr int32_t kUnexpandedRuleStartSequenceId = 128000; /*! * \brief A parent_id value of kNoParent means this ParserState is the root of the parsing stack. */ static constexpr int32_t kNoPrevInputPos = -1; /*! \brief A sequence_id value of kInvalid means the ParserState is invalid. */ static constexpr int32_t kInvalidSequenceId = -1; /*! \brief The rule's id. */ int32_t rule_id = -1; /*! \brief Which choice in this rule is selected. */ int32_t sequence_id = -1; /*! * \brief Which element of the choice sequence is to be visited. When the current sequence is * a tag dispatch rule, this element id is the current node. */ int32_t element_id = -1; /*! \brief The position of the state, i.e. from which position, the rule starts. */ int32_t rule_start_pos = -1; /*! \brief The id of the sub element in the current selement of the sequence. */ int32_t sub_element_id = 0; /*! \brief The number of times the element is repeated. It will be used in kRepeat.*/ int32_t repeat_count = 0; /*! \brief Partial codepoint accumulated during UTF-8 decoding for positive character classes. */ int32_t partial_codepoint = 0; /*! \brief The element is invalid when sequence_id is -1. */ bool IsInvalid() const { return sequence_id == -1; } static ParserState GetInvalidState() { return {-1, -1, -1, -1, -1}; } bool operator==(const ParserState& other) const { return rule_id == other.rule_id && sequence_id == other.sequence_id && element_id == other.element_id && sub_element_id == other.sub_element_id; } bool operator<(const ParserState& other) const { if (rule_id != other.rule_id) return rule_id < other.rule_id; if (sequence_id != other.sequence_id) return sequence_id < other.sequence_id; if (element_id != other.element_id) return element_id < other.element_id; if (rule_start_pos != other.rule_start_pos) return rule_start_pos < other.rule_start_pos; if (sub_element_id != other.sub_element_id) return sub_element_id < other.sub_element_id; return repeat_count < other.repeat_count; } friend std::ostream& operator<<(std::ostream& os, const ParserState& state) { os << state.ToString(); return os; } std::string ToString() const { std::string result = "ParserState(rule_id=" + std::to_string(rule_id) + ", sequence_id=" + std::to_string(sequence_id) + ", element_id=" + std::to_string(element_id) + ", rule_start_pos=" + std::to_string(rule_start_pos) + ", sub_element_id=" + std::to_string(sub_element_id); if (repeat_count != 0) { result += ", repeat_count=" + std::to_string(repeat_count); } if (partial_codepoint != 0) { result += ", partial_codepoint=" + std::to_string(partial_codepoint); } result += ")"; return result; } }; XGRAMMAR_MEMBER_ARRAY( ParserState, &ParserState::rule_id, &ParserState::sequence_id, &ParserState::element_id, &ParserState::rule_start_pos, &ParserState::sub_element_id, &ParserState::repeat_count, &ParserState::partial_codepoint ); /*! * \brief When getting the mask of the state, we don't need to consider the rule_start_pos. */ class StateHashForCache { public: size_t operator()(const ParserState& state) const { return HashCombine(state.rule_id, state.sequence_id, state.element_id, state.sub_element_id); } }; /*! * \brief When matching the state, we need to consider the rule_start_pos, since if two states * don't have the same rule_start_pos, they are not the same state. */ class StateEqualForParsing { public: bool operator()(const ParserState& lhs, const ParserState& rhs) const { return lhs.rule_id == rhs.rule_id && lhs.sequence_id == rhs.sequence_id && lhs.element_id == rhs.element_id && lhs.rule_start_pos == rhs.rule_start_pos && lhs.sub_element_id == rhs.sub_element_id && lhs.repeat_count == rhs.repeat_count && lhs.partial_codepoint == rhs.partial_codepoint; } }; /*! * \brief This class is used to hash the ParserState for parsing. * If two ParserStates don't have the same rule_start_pos, they are not the same state. */ class StateHashForParsing { public: size_t operator()(const ParserState& state) const { return HashCombine( state.rule_id, state.sequence_id, state.element_id, state.rule_start_pos, state.sub_element_id, state.repeat_count, state.partial_codepoint ); } }; /*! \brief This class is used to detect the repeated states. */ class RepeatDetector { private: const int transition_threshold_; std::vector visited_vector_; std::unordered_set visited_set_; int size_ = 0; public: RepeatDetector(const int transition_threshold = 50) : transition_threshold_(transition_threshold), size_(0) { visited_vector_.resize(transition_threshold_); } /*! * \brief Check if the element is visited. * \return True if visited, false otherwise. */ bool IsVisited(const ParserState& state) const; /*! * \brief Add the state into the visited states. * \param state The state to be added. */ void Insert(const ParserState& state); /*! \brief Reset the detector. */ void Clear(); }; class EarleyParser { /*! * \brief Here is an article about Earley Parser. * https://en.wikipedia.org/wiki/Earley_parser#Pseudocode * We divide the parser states into three categories: * - Scanable (which will be stored in scanable_state_history_). * - Predictable(If it predict a new rule successfully, then it will be stored in * rule_id_to_completable_states). * - completable(which can perform a completion operation). * A state will be stored in rule_id_to_completable_states_ if it can be completed, * and it will be stored in scanable_state_history_ if it can be scanned. Otherwise, * it will be discarded. */ protected: using GrammarExpr = Grammar::Impl::GrammarExpr; /*! \brief The grammar to be parsed. */ Grammar grammar_; /*! \brief In this round of advancing, check if the stop token can be accepted. */ bool tmp_accept_stop_token_ = false; /*! \brief store when accepting i characters, if the stop token can be accepted. */ std::vector is_completed_; /*! * \brief rule_id_to_completable_states[i][j] is the i pos j rule_id states. Earley * parser needs it to complete. */ Compact2DArray> rule_id_to_completable_states_; /*! * \brief The states history. state_stack[i] is a vector storing the states after accepting the * input[i-1]. */ Compact2DArray scanable_state_history_; /*! * \brief A temperate vector only used in Advance, used to add states in the * scanable_state_history. */ std::vector tmp_states_to_be_added_; /*! \brief It's the processing queue of the earley parser. */ std::queue tmp_process_state_queue_; /*! \brief The class is used to check if a state has been added into the queue. */ RepeatDetector tmp_states_visited_in_queue_; /*! \brief Check if the stop token is accepted. */ bool stop_token_is_accepted_ = false; /*! * \brief Check if the state has been added into the queue. * \param state The state to check. * \return True if in the vector, false otherwise. */ bool IsStateVisitedInQueue(const ParserState& state) const { return tmp_states_visited_in_queue_.IsVisited(state); } /*! * \brief The scanning operation of the Earley parser. Put the new states in the queue. */ void Scan(const ParserState& state, const uint8_t ch); /*! * \brief The completion operation of the Earley parser. * \param state The state to be completed. * \param debug_print Whether to print the debug information. * \details The reason is that if the state can't be scanned, then * add it into the next states is useless. Moreover, the end * of the grammar is used to check if the grammar is completed, * so it should be added into the next states. */ void Complete(const ParserState& state, bool debug_print = false); /*! * \brief The prediction operation of the Earley parser. * \param state The state to be predicted. * \param debug_print Whether to print the debug information. * \return First: If the state scanable, or the state is the end of the grammar, * then return true, otherwise return false. * \return Second: If the state is completable, then return true, otherwise return false. */ std::pair Predict(const ParserState& state, bool debug_print = false); /*! * \brief Handle the unexpanded rule, used for pushing initial state. * \param state The state to be handled. * \return True if the rule is unexpanded, false otherwise. */ bool ExpandAndEnqueueUnexpandedState(const ParserState& state); /*! * \brief Expand the rule, used for RuleRef and kTagDispatch. * \param state The state to be expanded, which is the parent state. * The type of the state is kTagDispatch or kSequence. Moreover, the * element of the sequence should be a rule reference; the node in * the kTagDispatch should be an end node. * \param grammar_expr The grammar expression to be expanded. * \param sub_grammar_expr The sub grammar expression to be expanded, especially * when the rule is a kSequence, and the sub rule is a kRuleRef. * \param debug_print Whether to print the debug information. */ void ExpandNextRuleRefElement( const ParserState& state, const GrammarExpr& grammar_expr, const GrammarExpr* sub_grammar_expr, bool debug_print = false ); /*! * \brief Expand the rule, used for RuleRef and kTagDispatch. * \param state The state to be expanded, and it's should be on the FSM. * \param debug_print Whether to print the debug information. */ void ExpandNextRuleRefElementOnFSM(const ParserState& state, bool debug_print = false); /*! * \brief Advance the parser to the next state, with the sub sequence is kCharacterClass. * \param state The state to be advanced. * \param ch The character to be advanced. * \param sub_sequence The sub sequence to be checked. * \return The next state, Invalid state if the character is not accepted. */ void AdvanceCharacterClass( const ParserState& state, const uint8_t ch, const GrammarExpr& sub_sequence ); /*! * \brief Advance the parser to the next state, with the sub sequence is kByteString. * \param state The state to be advanced. * \param ch The character to be advanced. * \param sub_sequence The sub sequence to be checked. * \return The next state, Invalid state if the character is not accepted. */ void AdvanceByteString( const ParserState& state, const uint8_t ch, const GrammarExpr& sub_sequence ); /*! * \brief Advance the parser to the next state, with the sub sequence is kCharacterClassStar. * \param state The state to be advanced. * \param ch The character to be advanced. * \param sub_sequence The sub sequence to be checked. * \return The next state, Invalid state if the character is not accepted. */ void AdvanceCharacterClassStar( const ParserState& state, const uint8_t ch, const GrammarExpr& sub_sequence ); /*! * \brief Advance the parser to the next state, with the sequence is kTagDispatch. * \param state The state to be advanced. * \param ch The character to be advanced. * \param cur_sequence The sequence of the current state. * \return The next state, Invalid state if the character is not accepted. */ void AdvanceFsm(const ParserState& state, const uint8_t ch); /*! * \brief Scan a token edge: check if token_id matches any kToken or kExcludeToken edge from * state. */ void ScanAtomicToken(const ParserState& state, int32_t token_id); /*! * \brief Advance the parser by accepting a whole token via kToken/kExcludeToken edges. * \param token_id The token ID to accept. * \param debug_print Whether to print debug info. * \return True if any state advanced, false otherwise. */ bool AdvanceAtomicToken(int32_t token_id, bool debug_print = false); /*! * \brief Enqueue the state into the queue. * \param state The state to be enqueued. * \details The state is enqueued if it is not visited in the queue. */ void Enqueue(const ParserState& state) { if (!IsStateVisitedInQueue(state)) { tmp_process_state_queue_.push(state); tmp_states_visited_in_queue_.Insert(state); } } /*! * \brief Enqueue the state into the queue, without prediction and completion. * \param state The state to be enqueued. */ void EnqueueWithoutProcessing(const ParserState& state) { if (!IsStateVisitedInQueue(state)) { tmp_states_visited_in_queue_.Insert(state); tmp_states_to_be_added_.push_back(state); } } public: /*! * \brief Constructor of the Earley parser. * \param grammar The grammar to be parsed. * \param initial_state The initial state to be pushed into the parser. */ EarleyParser( const Grammar& grammar, const ParserState& initial_state, const bool need_expand = true ); /*! * \brief From the current states, advance to the next state. * \param ch The character to be advanced. * \param debug_print Whether to print the debug information. * \return True if the character is accepted, false otherwise. * \note If the character isn't accepted, then the states won't be changed. */ bool Advance(const uint8_t ch, bool debug_print = false); /*! * \brief Remove the newly added states. * \param count The number of states to be removed. */ void PopLastStates(int32_t count = 1); /*! * \brief Check whether any of the multiple states stored in the parser has already completed. * \note Since the parser contains multiple parallel states, some may have already completed, * while others might still be able to accept more characters. * \return True if the root rule is completed, false otherwise. */ bool IsCompleted() const; /*! * \brief Push the initial state into the Earley parser. * \param state The initial state to be pushed. */ void PushStateAndExpand(const ParserState& state); /*! * \brief Reset the parser. * \note This function is used to reset the parser, and initialize the * parser with the root rule. */ void Reset(); /*! * \brief Get the current scanable states. * \return The scanable states. */ std::vector GetLatestScanableStates() const { std::vector latest_states; for (const auto& state : scanable_state_history_[scanable_state_history_.size() - 1]) { latest_states.push_back(state); } return latest_states; } /*! * \brief Push one state to check if it can accept the token. * \param state The state to be pushed. */ void PushOneStateToCheck(const ParserState& state) { rule_id_to_completable_states_.PushBack(std::vector>()); is_completed_.push_back(is_completed_.back()); scanable_state_history_.PushBack(&state, 1); return; } std::string PrintStates() const { std::string result; result += "There are " + std::to_string(scanable_state_history_.size()) + " steps in history. Last step: [\n"; for (const auto& state : scanable_state_history_[scanable_state_history_.size() - 1]) { result += state.ToString() + ", \n"; } result += "]"; return result; } }; } // namespace xgrammar #endif // XGRAMMAR_EARLEY_PARSER_H_ xgrammar-0.2.3/cpp/ebnf_script_creator.h000066400000000000000000000125271521764210300203120ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/ebnf_script_creator.h * \brief The header for the creating EBNF script. */ #ifndef XGRAMMAR_EBNF_SCRIPT_CREATOR_H_ #define XGRAMMAR_EBNF_SCRIPT_CREATOR_H_ #include #include #include #include #include #include "support/encoding.h" #include "support/logging.h" #include "support/utils.h" namespace xgrammar { /*! * \brief A class for creating EBNF grammar scripts. * * This class helps build EBNF (Extended Backus-Naur Form) grammar scripts * by managing rules and their content. */ class EBNFScriptCreator { public: /*! \brief Constructor */ EBNFScriptCreator() = default; /*! * \brief Adds a new rule to the grammar with a suggested name * \param rule_name_hint Suggested name for the rule * \param rule_body The EBNF content/definition of the rule * \return The actual name assigned to the rule */ std::string AddRule(const std::string& rule_name_hint, const std::string& rule_body) { return AddRuleWithAllocatedName(AllocateRuleName(rule_name_hint), rule_body); } /*! * \brief Generates a new rule name based on a suggested name * \param rule_name_hint Suggested name for the rule * \return The actual name assigned to the rule */ std::string AllocateRuleName(const std::string& rule_name_hint) { if (rule_names_.find(rule_name_hint) == rule_names_.end()) { rule_names_.insert(rule_name_hint); return rule_name_hint; } for (int i = 0; i < NAME_SUFFIX_MAXIMUM; ++i) { std::string rule_name = rule_name_hint + "_" + std::to_string(i); if (rule_names_.find(rule_name) == rule_names_.end()) { rule_names_.insert(rule_name); return rule_name; } } XGRAMMAR_LOG(FATAL) << "Cannot find a unique rule name for " << rule_name_hint; XGRAMMAR_UNREACHABLE(); } /*! * \brief Adds a new rule to the grammar with a allocated name. Used with AllocateRuleName() * \param rule_name The name of the rule to add * \param rule_body The EBNF content/definition of the rule * \return The actual name assigned to the rule */ std::string AddRuleWithAllocatedName(const std::string& rule_name, const std::string& rule_body) { XGRAMMAR_CHECK(rule_names_.find(rule_name) != rule_names_.end()) << "Rule name " << rule_name << " is not allocated"; rules_.emplace_back(rule_name, rule_body); return rule_name; } /*! * \brief Concatenates a list of strings with a space separator * \param items The list of strings to concatenate * \return The concatenated string */ static std::string Concat(const std::vector& items) { std::stringstream ss; ss << "("; for (int i = 0; i < static_cast(items.size()); ++i) { if (i > 0) { ss << " "; } ss << items[i]; } ss << ")"; return ss.str(); } /*! * \brief Joins a list of strings with an OR operator * \param items The list of strings to join * \return The joined string */ static std::string Or(const std::vector& items) { std::stringstream ss; ss << "("; for (int i = 0; i < static_cast(items.size()); ++i) { if (i > 0) { ss << " | "; } ss << items[i]; } ss << ")"; return ss.str(); } /*! * \brief Escape and quote a string * \param str The string to escape and quote * \return The escaped and quoted string */ static std::string Str(const std::string& str) { std::stringstream ss; ss << "\"" << EscapeString(str) << "\""; return ss.str(); } /*! * \brief Repeats an item a given number of times * \param item The item to repeat * \param min The minimum number of times to repeat the item * \param max The maximum number of times to repeat the item * \return The repeated string */ static std::string Repeat(const std::string& item, int min, int max) { std::stringstream ss; ss << item; if (min == 0 && max == 1) { ss << "?"; } else if (min == 0 && max == -1) { ss << "*"; } else if (min == 1 && max == -1) { ss << "+"; } else if (min == 0 && max == 0) { return ""; } else if (min == max) { ss << "{" << min << "}"; } else if (max == -1) { ss << "{" << min << ",}"; } else { ss << "{" << min << "," << max << "}"; } return ss.str(); } /*! * \brief Gets the complete EBNF grammar script * \return The full EBNF grammar script as a string */ std::string GetScript() { std::string script = ""; for (const auto& rule : rules_) { script += rule.first + " ::= " + rule.second + "\n"; } return script; } /*! * \brief Retrieves the content/definition of a specific rule * \param rule_name The name of the rule to look up * \return The EBNF content/definition of the specified rule */ std::string GetRuleContent(const std::string& rule_name) { auto it = std::find_if(rules_.begin(), rules_.end(), [rule_name](const auto& rule) { return rule.first == rule_name; }); if (it != rules_.end()) { return it->second; } return ""; } private: std::vector> rules_; std::unordered_set rule_names_; const int NAME_SUFFIX_MAXIMUM = 10000; }; } // namespace xgrammar #endif // XGRAMMAR_EBNF_SCRIPT_CREATOR_H_ xgrammar-0.2.3/cpp/fsm.cc000066400000000000000000002067261521764210300152260ustar00rootroot00000000000000/*! * Copyright (c) 2025 by Contributors * \file xgrammar/fsm.cc */ #include "fsm.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "support/encoding.h" #include "support/json_serializer.h" #include "support/logging.h" #include "support/reflection.h" #include "support/union_find_set.h" #include "support/utils.h" #include "xgrammar/exception.h" namespace xgrammar { /****************** FSMImplBase ******************/ template class FSMImplBase { static_assert( std::is_same_v>> || std::is_same_v>, "ContainerType must be std::vector> or Compact2DArray" ); public: /*! \brief Default constructor. */ FSMImplBase() = default; FSMImplBase(const ContainerType& edges, std::vector edge_aux_data = {}) : edges_(edges), edge_aux_data_(std::move(edge_aux_data)) {} FSMImplBase(ContainerType&& edges, std::vector edge_aux_data = {}) : edges_(std::move(edges)), edge_aux_data_(std::move(edge_aux_data)) {} int NumStates() const { return edges_.size(); } std::string EdgesToString(std::optional> states = std::nullopt) const; const ContainerType& GetEdges() const { return edges_; } // For std::vector>, return const std::vector& to avoid copying. // For Compact2DArray, return Compact2DArray::Row since it is just a simple // pointer. decltype(auto) GetEdges(int state) const { return edges_[state]; } void GetEpsilonClosure(std::unordered_set* state_set) const; void GetPossibleRules(int state_num, std::unordered_set* rules) const; void GetReachableStates(const std::vector& from, std::unordered_set* result) const; const std::vector& GetEdgeAuxData() const { return edge_aux_data_; } void SetEdgeAuxData(std::vector data) { edge_aux_data_ = std::move(data); } RepeatEdgeRef GetRepeatEdgeInfo(int32_t idx) const { return {edge_aux_data_.data() + idx}; } TokenEdgeRef GetTokenEdgeInfo(int32_t idx) const { return {edge_aux_data_.data() + idx}; } ExcludeTokenEdgeRef GetExcludeTokenEdgeInfo(int32_t idx) const { return {edge_aux_data_.data() + idx}; } protected: ContainerType edges_; std::vector edge_aux_data_; friend struct member_trait; }; template std::string FSMImplBase::EdgesToString(std::optional> states ) const { std::string result = "[\n"; auto f_print_one = [&, this](int i) { result += std::to_string(i) + ": ["; const auto& edges = edges_[i]; for (int j = 0; j < static_cast(edges.size()); ++j) { const auto& edge = edges[j]; if (edge.min >= 0 && edge.min != edge.max) { std::string char_min_str = EscapeString(static_cast(edge.min)); std::string char_max_str = EscapeString(static_cast(edge.max)); result += "[" + char_min_str + "-" + char_max_str + "]->" + std::to_string(edge.target); } else if (edge.min >= 0 && edge.min == edge.max) { std::string char_str = EscapeString(static_cast(edge.min)); result += "'" + char_str + "'->" + std::to_string(edge.target); } else if (edge.min == FSMEdge::EdgeType::kRuleRef) { result += "Rule(" + std::to_string(edge.max) + ")->" + std::to_string(edge.target); } else if (edge.min == FSMEdge::EdgeType::kEpsilon) { result += "Eps->" + std::to_string(edge.target); } else if (edge.min == FSMEdge::EdgeType::kEOS) { result += "EOS->" + std::to_string(edge.target); } else if (edge.min == FSMEdge::EdgeType::kRepeatRef) { auto info = GetRepeatEdgeInfo(edge.max); result += "Repeat(rule=" + std::to_string(info.RuleId()) + ", min=" + std::to_string(info.Lower()) + ", max=" + std::to_string(info.Upper()) + ")->" + std::to_string(edge.target); } else if (edge.min == FSMEdge::EdgeType::kToken) { auto info = GetTokenEdgeInfo(edge.max); result += "Token("; for (int32_t k = 0; k < info.Count(); ++k) { if (k > 0) result += ", "; result += std::to_string(info.TokenIds()[k]); } result += ")->" + std::to_string(edge.target); } else if (edge.min == FSMEdge::EdgeType::kExcludeToken) { auto info = GetExcludeTokenEdgeInfo(edge.max); result += "ExcludeToken("; for (int32_t k = 0; k < info.Count(); ++k) { if (k > 0) result += ", "; result += std::to_string(info.TokenIds()[k]); } result += ")->" + std::to_string(edge.target); } if (j < static_cast(edges.size()) - 1) { result += ", "; } } result += "]\n"; }; if (states.has_value()) { for (int i : states.value()) { f_print_one(i); } } else { for (int i = 0; i < int(NumStates()); ++i) { f_print_one(i); } } result += "]"; return result; } template void FSMImplBase::GetEpsilonClosure(std::unordered_set* state_set) const { std::queue queue; for (const auto& state : *state_set) { queue.push(state); } while (!queue.empty()) { int current = queue.front(); queue.pop(); for (const auto& edge : edges_[current]) { if (!edge.IsEpsilon()) { continue; } if (state_set->find(edge.target) != state_set->end()) { continue; } state_set->insert(edge.target); queue.push(edge.target); } } } template void FSMImplBase::GetPossibleRules(int state, std::unordered_set* rules) const { rules->clear(); for (const auto& edge : edges_[state]) { if (edge.IsRuleRef()) { rules->insert(edge.GetRefRuleId()); } } } template void FSMImplBase::GetReachableStates( const std::vector& from, std::unordered_set* result ) const { result->clear(); std::queue queue; for (const auto& state : from) { queue.push(state); result->insert(state); } while (!queue.empty()) { int current = queue.front(); queue.pop(); for (const auto& edge : edges_[current]) { if (result->find(edge.target) != result->end()) { continue; } result->insert(edge.target); queue.push(edge.target); } } } /****************** FSM::Impl ******************/ class FSM::Impl : public FSMImplBase>> { using EdgeType = FSMEdge::EdgeType; public: Impl() = default; Impl(int num_states = 0) { edges_.resize(num_states); } using FSMImplBase>>::FSMImplBase; int GetNextState(int from, int value, EdgeType edge_type) const; using FSMImplBase>>::GetEdges; std::vector>& GetEdges() { return edges_; } std::vector& GetEdges(int state) { return edges_[state]; } void Advance( const std::unordered_set& from, int value, std::unordered_set* result, EdgeType edge_type, bool from_is_closure ) const; int AddState() { edges_.emplace_back(); return edges_.size() - 1; } void AddEdge(int from, int to, int32_t min, int32_t max) { XGRAMMAR_DCHECK(from < static_cast(edges_.size())); edges_[from].push_back({min, max, to}); } void AddRuleEdge(int from, int to, int32_t rule_id) { AddEdge(from, to, FSMEdge::EdgeType::kRuleRef, rule_id); } void AddEpsilonEdge(int from, int to) { AddEdge(from, to, FSMEdge::EdgeType::kEpsilon, 0); } void AddEOSEdge(int from, int to) { AddEdge(from, to, FSMEdge::EdgeType::kEOS, 0); } void AddRepeatEdge(int from, int to, int32_t rule_id, int32_t lower, int32_t upper) { XGRAMMAR_DCHECK(edges_[from].empty()) << "A state with a kRepeatRef edge must have no other outgoing edges."; XGRAMMAR_DCHECK(edge_aux_data_.size() <= INT32_MAX); int32_t aux_index = static_cast(edge_aux_data_.size()); edge_aux_data_.reserve(edge_aux_data_.size() + 3); edge_aux_data_.emplace_back(rule_id); edge_aux_data_.emplace_back(lower); edge_aux_data_.emplace_back(upper); AddEdge(from, to, FSMEdge::EdgeType::kRepeatRef, aux_index); } void AddTokenEdge(int from, int to, const std::vector& token_ids) { XGRAMMAR_DCHECK(!token_ids.empty()) << "Token set must not be empty"; XGRAMMAR_CHECK(edge_aux_data_.size() <= INT32_MAX) << "edge_aux_data_ overflow: too many auxiliary data entries"; int32_t aux_index = static_cast(edge_aux_data_.size()); edge_aux_data_.push_back(static_cast(token_ids.size())); for (int32_t id : token_ids) { edge_aux_data_.push_back(id); } edges_[from].push_back(FSMEdge(FSMEdge::EdgeType::kToken, aux_index, to)); } void AddExcludeTokenEdge(int from, int to, const std::vector& token_ids) { XGRAMMAR_DCHECK(!token_ids.empty()) << "Token exclude set must not be empty"; XGRAMMAR_CHECK(edge_aux_data_.size() <= INT32_MAX) << "edge_aux_data_ overflow: too many auxiliary data entries"; int32_t aux_index = static_cast(edge_aux_data_.size()); edge_aux_data_.push_back(static_cast(token_ids.size())); for (int32_t id : token_ids) { edge_aux_data_.push_back(id); } edges_[from].push_back(FSMEdge(FSMEdge::EdgeType::kExcludeToken, aux_index, to)); } void AddFSM(const FSM& fsm, std::vector* state_mapping); FSM RebuildWithMapping(const std::vector& state_mapping, int new_num_states) const; void SortEdges(); CompactFSM ToCompact(); friend class FSMWithStartEnd; }; int FSM::Impl::GetNextState(int from, int value, EdgeType edge_type) const { XGRAMMAR_DCHECK(edge_type != EdgeType::kEpsilon) << "Should not call GetNextState with edge type kEpsilon."; if (edge_type == EdgeType::kCharRange) { for (const auto& edge : edges_[from]) { if (edge.min >= EdgeType::kCharRange && edge.min <= value && edge.max >= value) { return edge.target; } } return FSM::kNoNextState; } else if (edge_type == EdgeType::kRuleRef) { for (const auto& edge : edges_[from]) { if (edge.min == EdgeType::kRuleRef && edge.GetRefRuleId() == value) { return edge.target; } } return FSM::kNoNextState; } else if (edge_type == EdgeType::kEOS) { for (const auto& edge : edges_[from]) { if (edge.min == EdgeType::kEOS) { return edge.target; } } return FSM::kNoNextState; } else if (edge_type == EdgeType::kRepeatRef) { // By invariant, a state with kRepeatRef has exactly one outgoing edge. XGRAMMAR_DCHECK(edges_[from].size() == 1 && edges_[from][0].IsRepeatRef()); return edges_[from][0].target; } else { XGRAMMAR_DCHECK(false) << "Invalid edge type: " << static_cast(edge_type); } XGRAMMAR_UNREACHABLE(); } void FSM::Impl::Advance( const std::unordered_set& from, int value, std::unordered_set* result, EdgeType edge_type, bool from_is_closure ) const { XGRAMMAR_DCHECK(edge_type != EdgeType::kEpsilon) << "Should not call Advance with edge type kEpsilon."; const std::unordered_set* start_closure; std::unordered_set start_closure_tmp; if (from_is_closure) { start_closure = &from; } else { start_closure_tmp.insert(from.begin(), from.end()); GetEpsilonClosure(&start_closure_tmp); start_closure = &start_closure_tmp; } result->clear(); if (edge_type == EdgeType::kCharRange) { for (const auto& state : *start_closure) { for (const auto& edge : edges_[state]) { if (edge.IsCharRange() && edge.min <= value && edge.max >= value) { result->insert(edge.target); } } } } else if (edge_type == EdgeType::kRuleRef) { for (const auto& state : *start_closure) { for (const auto& edge : edges_[state]) { if (edge.IsRuleRef() && edge.GetRefRuleId() == value) { result->insert(edge.target); } } } } else if (edge_type == EdgeType::kEOS) { for (const auto& state : *start_closure) { for (const auto& edge : edges_[state]) { if (edge.IsEOS()) { result->insert(edge.target); } } } } else if (edge_type == EdgeType::kRepeatRef) { // By invariant, a state with kRepeatRef has exactly one outgoing edge. for (const auto& state : *start_closure) { if (!edges_[state].empty() && edges_[state][0].IsRepeatRef()) { result->insert(edges_[state][0].target); } } } else { XGRAMMAR_DCHECK(false) << "Invalid edge type: " << static_cast(edge_type); } // Get the epsilon closure of the result. GetEpsilonClosure(result); } void FSM::Impl::AddFSM(const FSM& fsm, std::vector* state_mapping) { int old_num_states = NumStates(); int32_t aux_offset = static_cast(edge_aux_data_.size()); const auto& other_aux = fsm.GetEdgeAuxData(); edge_aux_data_.insert(edge_aux_data_.end(), other_aux.begin(), other_aux.end()); if (state_mapping != nullptr) { state_mapping->clear(); state_mapping->reserve(fsm.NumStates()); for (int i = 0; i < fsm.NumStates(); ++i) { state_mapping->push_back(i + old_num_states); } } edges_.resize(edges_.size() + fsm.NumStates()); for (int i = 0; i < fsm.NumStates(); ++i) { for (const auto& edge : fsm.GetEdges()[i]) { int32_t max_val = edge.max; if (edge.IsAuxEdge() && aux_offset > 0) { max_val = static_cast(edge.max + aux_offset); } AddEdge(i + old_num_states, edge.target + old_num_states, edge.min, max_val); } } } FSM FSM::Impl::RebuildWithMapping(const std::vector& state_mapping, int new_num_states) const { std::vector> new_edges(new_num_states); for (int i = 0; i < static_cast(edges_.size()); ++i) { for (const auto& edge : edges_[i]) { if (edge.IsEpsilon() && state_mapping[i] == state_mapping[edge.target]) { continue; // Skip self-loops for epsilon edges. } new_edges[state_mapping[i]].emplace_back(edge.min, edge.max, state_mapping[edge.target]); } } // aux_indices remain stable since only state ids are remapped for (int i = 0; i < new_num_states; ++i) { std::sort(new_edges[i].begin(), new_edges[i].end()); const auto& end_iter = std::unique(new_edges[i].begin(), new_edges[i].end()); new_edges[i].erase(end_iter, new_edges[i].end()); } return FSM(std::move(new_edges), std::vector(edge_aux_data_)); } void FSM::Impl::SortEdges() { for (int i = 0; i < static_cast(edges_.size()); ++i) { std::sort(edges_[i].begin(), edges_[i].end()); } } CompactFSM FSM::Impl::ToCompact() { SortEdges(); Compact2DArray edges; for (int i = 0; i < static_cast(edges_.size()); ++i) { edges.PushBack(edges_[i]); } return CompactFSM(std::move(edges), std::move(edge_aux_data_)); } /****************** FSM ******************/ FSM::FSM(int num_states) : pimpl_(std::make_shared(num_states)) {} FSM::FSM(const std::vector>& edges, std::vector edge_aux_data) : pimpl_(std::make_shared(edges, std::move(edge_aux_data))) {} FSM::FSM(std::vector>&& edges, std::vector edge_aux_data) : pimpl_(std::make_shared(std::move(edges), std::move(edge_aux_data))) {} int FSM::NumStates() const { return pimpl_->NumStates(); } int FSM::AddState() { return pimpl_->AddState(); } void FSM::AddEdge(int from, int to, int32_t min, int32_t max) { pimpl_->AddEdge(from, to, min, max); } void FSM::AddEdge(int from, int to, FSMEdge::EdgeType type, int32_t value) { pimpl_->AddEdge(from, to, type, value); } void FSM::AddEpsilonEdge(int from, int to) { pimpl_->AddEpsilonEdge(from, to); } void FSM::AddRuleEdge(int from, int to, int32_t rule_id) { pimpl_->AddRuleEdge(from, to, rule_id); } void FSM::AddEOSEdge(int from, int to) { pimpl_->AddEOSEdge(from, to); } void FSM::AddRepeatEdge(int from, int to, int32_t rule_id, int32_t lower, int32_t upper) { pimpl_->AddRepeatEdge(from, to, rule_id, lower, upper); } void FSM::AddTokenEdge(int from, int to, const std::vector& token_ids) { pimpl_->AddTokenEdge(from, to, token_ids); } void FSM::AddExcludeTokenEdge(int from, int to, const std::vector& token_ids) { pimpl_->AddExcludeTokenEdge(from, to, token_ids); } const std::vector& FSM::GetEdgeAuxData() const { return pimpl_->GetEdgeAuxData(); } void FSM::SetEdgeAuxData(std::vector data) { pimpl_->SetEdgeAuxData(std::move(data)); } RepeatEdgeRef FSM::GetRepeatEdgeInfo(int32_t idx) const { return pimpl_->GetRepeatEdgeInfo(idx); } TokenEdgeRef FSM::GetTokenEdgeInfo(int32_t idx) const { return pimpl_->GetTokenEdgeInfo(idx); } ExcludeTokenEdgeRef FSM::GetExcludeTokenEdgeInfo(int32_t idx) const { return pimpl_->GetExcludeTokenEdgeInfo(idx); } void FSM::AddFSM(const FSM& fsm, std::vector* state_mapping) { pimpl_->AddFSM(fsm, state_mapping); } std::string FSM::EdgesToString(std::optional> states) const { return pimpl_->EdgesToString(states); } const std::vector& FSM::GetEdges(int state) const { return pimpl_->GetEdges(state); } std::vector>& FSM::GetEdges() { return pimpl_->GetEdges(); } const std::vector>& FSM::GetEdges() const { return pimpl_->GetEdges(); } std::vector& FSM::GetEdges(int state) { return pimpl_->GetEdges(state); } FSM FSM::Copy() const { return FSM(std::make_shared(*pimpl_)); } int FSM::GetNextState(int from, int value, FSMEdge::EdgeType edge_type) const { return pimpl_->GetNextState(from, value, edge_type); } void FSM::Advance( const std::unordered_set& from, int value, std::unordered_set* result, FSMEdge::EdgeType edge_type, bool from_is_closure ) const { pimpl_->Advance(from, value, result, edge_type, from_is_closure); } void FSM::GetPossibleRules(int state, std::unordered_set* rules) const { pimpl_->GetPossibleRules(state, rules); } void FSM::GetEpsilonClosure(std::unordered_set* state_set) const { pimpl_->GetEpsilonClosure(state_set); } void FSM::GetReachableStates(const std::vector& from, std::unordered_set* result) const { pimpl_->GetReachableStates(from, result); } FSM FSM::RebuildWithMapping(const std::vector& state_mapping, int new_num_states) const { return pimpl_->RebuildWithMapping(state_mapping, new_num_states); } void FSM::SortEdges() { pimpl_->SortEdges(); } CompactFSM FSM::ToCompact() { return pimpl_->ToCompact(); } /****************** CompactFSM::Impl ******************/ class CompactFSM::Impl : public FSMImplBase> { using EdgeType = FSMEdge::EdgeType; public: Impl() = default; Impl(const Compact2DArray& edges, std::vector edge_aux_data = {}) : FSMImplBase>(edges, std::move(edge_aux_data)), edge_num_(ComputeEdgeNum(edges_)) {} Impl(Compact2DArray&& edges, std::vector edge_aux_data = {}) : FSMImplBase>(std::move(edges), std::move(edge_aux_data)), edge_num_(ComputeEdgeNum(edges_)) {} void GetNextStates(int from, int value, EdgeType edge_type, std::vector* target) const; void Advance( const std::unordered_set& from, int value, std::unordered_set* result, FSMEdge::EdgeType edge_type, bool from_is_closure ) const; FSM ToFSM() const; size_t GetNumEdges() const { return edge_num_; } size_t edge_num_ = 0; friend std::size_t MemorySize(const Impl& impl) { return MemorySize(impl.edges_) + MemorySize(impl.edge_aux_data_) + sizeof(impl.edge_num_); } private: static size_t ComputeEdgeNum(const Compact2DArray& edges) { size_t edge_num = 0; for (int i = 0; i < edges.size(); ++i) { edge_num += edges[i].size(); } return edge_num; } }; XGRAMMAR_MEMBER_TABLE( CompactFSM::Impl, "edges", &CompactFSM::Impl::edges_, "edge_aux_data", &CompactFSM::Impl::edge_aux_data_, "edge_num", &CompactFSM::Impl::edge_num_ ); void CompactFSM::Impl::GetNextStates( int from, int value, EdgeType edge_type, std::vector* targets ) const { targets->clear(); XGRAMMAR_DCHECK(edge_type != EdgeType::kEpsilon) << "Should not call GetNextState with edge type kEpsilon."; if (edge_type == EdgeType::kCharRange) { for (const auto& edge : edges_[from]) { if (edge.min < EdgeType::kCharRange) { continue; } else if (edge.min > value) { break; } else if (edge.max >= value) { targets->push_back(edge.target); } } } else if (edge_type == EdgeType::kRuleRef) { for (const auto& edge : edges_[from]) { if (edge.min < EdgeType::kRuleRef) { continue; } else if (edge.min > EdgeType::kRuleRef) { break; } else if (edge.GetRefRuleId() == value) { targets->push_back(edge.target); } } } else if (edge_type == EdgeType::kEOS) { for (const auto& edge : edges_[from]) { if (edge.min < EdgeType::kEOS) { continue; } else if (edge.min > EdgeType::kEOS) { break; } else if (edge.max >= EdgeType::kEOS) { targets->push_back(edge.target); } } } else if (edge_type == EdgeType::kRepeatRef) { // By invariant, a state with kRepeatRef has exactly one outgoing edge. for (const auto& edge : edges_[from]) { if (edge.IsRepeatRef()) { targets->push_back(edge.target); break; } } } else { XGRAMMAR_DCHECK(false) << "Invalid edge type: " << static_cast(edge_type); } } void CompactFSM::Impl::Advance( const std::unordered_set& from, int value, std::unordered_set* result, FSMEdge::EdgeType edge_type, bool from_is_closure ) const { const std::unordered_set* start_closure; std::unordered_set start_closure_tmp; if (from_is_closure) { start_closure = &from; } else { start_closure_tmp.insert(from.begin(), from.end()); GetEpsilonClosure(&start_closure_tmp); start_closure = &start_closure_tmp; } result->clear(); if (edge_type == EdgeType::kCharRange) { for (const auto& state : *start_closure) { for (const auto& edge : edges_[state]) { if (edge.min < EdgeType::kCharRange) { continue; } else if (edge.min > value) { break; } else if (edge.max >= value) { result->insert(edge.target); } } } } else if (edge_type == EdgeType::kRuleRef) { for (const auto& state : *start_closure) { for (const auto& edge : edges_[state]) { if (edge.min < EdgeType::kRuleRef) { continue; } else if (edge.min > EdgeType::kRuleRef) { break; } else if (edge.GetRefRuleId() == value) { result->insert(edge.target); } } } } else if (edge_type == EdgeType::kEOS) { for (const auto& state : *start_closure) { for (const auto& edge : edges_[state]) { if (edge.min < EdgeType::kEOS) { continue; } else if (edge.min > EdgeType::kEOS) { break; } else if (edge.max >= EdgeType::kEOS) { result->insert(edge.target); } } } } else if (edge_type == EdgeType::kRepeatRef) { // By invariant, a state with kRepeatRef has exactly one outgoing edge. for (const auto& state : *start_closure) { for (const auto& edge : edges_[state]) { if (edge.IsRepeatRef()) { result->insert(edge.target); break; } } } } else { XGRAMMAR_DCHECK(false) << "Invalid edge type: " << static_cast(edge_type); } // Get the epsilon closure of the result. GetEpsilonClosure(result); } FSM CompactFSM::Impl::ToFSM() const { std::vector> edges(NumStates()); for (int i = 0; i < edges_.size(); i++) { const auto& row = edges_[i]; edges[i].insert(edges[i].end(), row.begin(), row.end()); } return FSM(std::move(edges), std::vector(edge_aux_data_)); } /****************** CompactFSM ******************/ CompactFSM::CompactFSM(const Compact2DArray& edges, std::vector edge_aux_data) : pimpl_(std::make_shared(edges, std::move(edge_aux_data))) {} CompactFSM::CompactFSM(Compact2DArray&& edges, std::vector edge_aux_data) : pimpl_(std::make_shared(std::move(edges), std::move(edge_aux_data))) {} int CompactFSM::NumStates() const { return pimpl_->NumStates(); } const Compact2DArray& CompactFSM::GetEdges() const { return pimpl_->GetEdges(); } Compact2DArray::Row CompactFSM::GetEdges(int state) const { return pimpl_->GetEdges(state); } std::string CompactFSM::EdgesToString(std::optional> states) const { return pimpl_->EdgesToString(states); } void CompactFSM::GetNextStates( int from, int value, FSMEdge::EdgeType edge_type, std::vector* targets ) const { return pimpl_->GetNextStates(from, value, edge_type, targets); } void CompactFSM::Advance( const std::unordered_set& from, int value, std::unordered_set* result, FSMEdge::EdgeType edge_type, bool from_is_closure ) const { pimpl_->Advance(from, value, result, edge_type, from_is_closure); } void CompactFSM::GetPossibleRules(int state_num, std::unordered_set* rules) const { pimpl_->GetPossibleRules(state_num, rules); } void CompactFSM::GetEpsilonClosure(std::unordered_set* state_set) const { pimpl_->GetEpsilonClosure(state_set); } void CompactFSM::GetReachableStates(const std::vector& from, std::unordered_set* result) const { pimpl_->GetReachableStates(from, result); } size_t CompactFSM::GetNumEdges() const { return pimpl_->GetNumEdges(); } FSM CompactFSM::ToFSM() const { return pimpl_->ToFSM(); } const std::vector& CompactFSM::GetEdgeAuxData() const { return pimpl_->GetEdgeAuxData(); } void CompactFSM::SetEdgeAuxData(std::vector data) { pimpl_->SetEdgeAuxData(std::move(data)); } RepeatEdgeRef CompactFSM::GetRepeatEdgeInfo(int32_t idx) const { return pimpl_->GetRepeatEdgeInfo(idx); } TokenEdgeRef CompactFSM::GetTokenEdgeInfo(int32_t idx) const { return pimpl_->GetTokenEdgeInfo(idx); } ExcludeTokenEdgeRef CompactFSM::GetExcludeTokenEdgeInfo(int32_t idx) const { return pimpl_->GetExcludeTokenEdgeInfo(idx); } picojson::value SerializeJSONValue(const CompactFSM& value) { return detail::json_serializer::AutoSerializeJSONValuePImpl(value); } std::optional DeserializeJSONValue( CompactFSM* result, const picojson::value& value, const std::string& type_name ) { return detail::json_serializer::AutoDeserializeJSONValuePImpl(result, value, type_name); } struct CompactFSMWithStartEndSerializeHelper { CompactFSM fsm; int start; bool is_dfa; std::vector end_index; size_t edge_num; CompactFSMWithStartEndSerializeHelper(const CompactFSMWithStartEnd& compact_fsm_with_se) : fsm(compact_fsm_with_se.fsm_), start(compact_fsm_with_se.start_), is_dfa(compact_fsm_with_se.is_dfa_), edge_num(compact_fsm_with_se.edge_num_) { end_index.reserve(compact_fsm_with_se.NumStates()); for (int i = 0; i < static_cast(compact_fsm_with_se.ends_.size()); ++i) { if (compact_fsm_with_se.ends_[i]) { end_index.push_back(i); } } } CompactFSMWithStartEndSerializeHelper() = default; }; XGRAMMAR_MEMBER_ARRAY( CompactFSMWithStartEndSerializeHelper, &CompactFSMWithStartEndSerializeHelper::fsm, &CompactFSMWithStartEndSerializeHelper::start, &CompactFSMWithStartEndSerializeHelper::end_index, &CompactFSMWithStartEndSerializeHelper::is_dfa, &CompactFSMWithStartEndSerializeHelper::edge_num ); picojson::value SerializeJSONValue(const CompactFSMWithStartEnd& value) { return AutoSerializeJSONValue(CompactFSMWithStartEndSerializeHelper(value)); } std::optional DeserializeJSONValue( CompactFSMWithStartEnd* result, const picojson::value& value, const std::string& type_name ) { CompactFSMWithStartEndSerializeHelper tmp; auto err = AutoDeserializeJSONValue(&tmp, value, type_name); if (err.has_value()) { return err; } result->fsm_ = std::move(tmp.fsm); result->start_ = tmp.start; result->is_dfa_ = tmp.is_dfa; result->edge_num_ = tmp.edge_num; const auto& end_index = tmp.end_index; result->ends_.resize(result->fsm_.NumStates(), false); for (const auto& idx : end_index) { result->ends_[idx] = true; } return std::nullopt; } struct CompactFSMWithStartEndWithSizeSerializeHelper { CompactFSMWithStartEnd fsm; size_t edge_num; size_t node_num; CompactFSMWithStartEndWithSizeSerializeHelper( const CompactFSMWithStartEndWithSize& compact_fsm_with_size ) : fsm(compact_fsm_with_size.fsm_), edge_num(compact_fsm_with_size.edge_num_), node_num(compact_fsm_with_size.node_num_) {} CompactFSMWithStartEndWithSizeSerializeHelper() = default; }; XGRAMMAR_MEMBER_ARRAY( CompactFSMWithStartEndWithSizeSerializeHelper, &CompactFSMWithStartEndWithSizeSerializeHelper::fsm, &CompactFSMWithStartEndWithSizeSerializeHelper::edge_num, &CompactFSMWithStartEndWithSizeSerializeHelper::node_num ); picojson::value SerializeJSONValue(const CompactFSMWithStartEndWithSize& value) { return AutoSerializeJSONValue(CompactFSMWithStartEndWithSizeSerializeHelper(value)); } std::optional DeserializeJSONValue( CompactFSMWithStartEndWithSize* result, const picojson::value& value, const std::string& type_name ) { CompactFSMWithStartEndWithSizeSerializeHelper tmp; auto err = AutoDeserializeJSONValue(&tmp, value, type_name); if (err.has_value()) { return err; } result->fsm_ = std::move(tmp.fsm); result->edge_num_ = tmp.edge_num; result->node_num_ = tmp.node_num; return std::nullopt; } /****************** FSMWithStartEnd ******************/ std::string FSMWithStartEnd::ToString() const { std::string result; result += "FSM(num_states=" + std::to_string(NumStates()) + ", start=" + std::to_string(start_) + ", end=["; std::unordered_set reachable_states; GetReachableStates(&reachable_states); std::vector reachable_states_vec(reachable_states.begin(), reachable_states.end()); std::sort(reachable_states_vec.begin(), reachable_states_vec.end()); bool first = true; for (int i = 0; i < NumStates(); ++i) { if (!IsEndState(i)) { continue; } if (!first) { result += ", "; } first = false; result += std::to_string(i); } result += "], edges=" + fsm_.EdgesToString(reachable_states_vec) + ")"; return result; } std::ostream& operator<<(std::ostream& os, const FSMWithStartEnd& fsm) { os << fsm.ToString(); return os; } FSMWithStartEnd FSMWithStartEnd::Copy() const { return FSMWithStartEnd(fsm_.Copy(), start_, ends_, is_dfa_); } FSMWithStartEnd FSMWithStartEnd::RebuildWithMapping( const std::vector& state_mapping, int new_num_states ) const { FSM new_fsm = fsm_.RebuildWithMapping(state_mapping, new_num_states); auto new_start = state_mapping[start_]; std::vector new_ends(new_num_states, false); for (int end = 0; end < NumStates(); ++end) { if (IsEndState(end)) { new_ends[state_mapping[end]] = true; } } return FSMWithStartEnd(new_fsm, new_start, new_ends); } CompactFSMWithStartEnd FSMWithStartEnd::ToCompact() { return CompactFSMWithStartEnd(fsm_.ToCompact(), start_, ends_, is_dfa_); } FSMWithStartEndWithSize FSMWithStartEnd::AddToCompleteFSM( FSM* complete_fsm, std::vector* state_mapping ) { XGRAMMAR_DCHECK(state_mapping != nullptr) << "state_mapping cannot be nullptr"; complete_fsm->AddFSM(fsm_, state_mapping); int new_start = (*state_mapping)[start_]; std::vector new_ends(complete_fsm->NumStates(), false); for (int end = 0; end < NumStates(); ++end) { if (IsEndState(end)) { new_ends[(*state_mapping)[end]] = true; } } int num_edges = 0; for (int i = 0; i < fsm_.NumStates(); ++i) { num_edges += fsm_.GetEdges(i).size(); } int num_nodes = fsm_.NumStates(); auto fsm_with_se = FSMWithStartEnd(*complete_fsm, new_start, new_ends, is_dfa_); return FSMWithStartEndWithSize(fsm_with_se, num_edges, num_nodes); } FSMWithStartEnd FSMWithStartEnd::Star() const { FSM fsm = fsm_.Copy(); auto new_start = fsm.AddState(); for (int end = 0; end < NumStates(); ++end) { if (IsEndState(end)) { fsm.AddEpsilonEdge(end, new_start); } } fsm.AddEpsilonEdge(new_start, start_); std::vector is_end(NumStates() + 1, false); is_end[new_start] = true; return FSMWithStartEnd(fsm, new_start, is_end); } FSMWithStartEnd FSMWithStartEnd::Plus() const { FSM fsm = fsm_.Copy(); for (int end = 0; end < NumStates(); ++end) { if (IsEndState(end)) { fsm.AddEpsilonEdge(end, start_); } } return FSMWithStartEnd(fsm, start_, ends_); } FSMWithStartEnd FSMWithStartEnd::Optional() const { FSM fsm = fsm_.Copy(); for (int end = 0; end < NumStates(); ++end) { if (IsEndState(end)) { fsm.AddEpsilonEdge(start_, end); break; } } return FSMWithStartEnd(fsm, start_, ends_); } Result FSMWithStartEnd::Not(int max_result_num_states) const { // Check if the FSM contains any rule references. if (!IsLeaf()) { XGRAMMAR_LOG(FATAL) << "Not operation is not supported for FSM with rule references."; } FSMWithStartEnd result; if (is_dfa_) { result = Copy(); } else { Result dfa_result = ToDFA(max_result_num_states); if (dfa_result.IsErr()) { return dfa_result; } result = std::move(dfa_result).Unwrap(); } // Reverse all the final states. std::vector new_final_states(result.NumStates() + 1, false); for (int i = 0; i < result.NumStates(); ++i) { if (!result.IsEndState(i)) { new_final_states[i] = true; // Mark all states as final except the original final states. } } // Add a new final state that accepts all characters. int accept_all_new_state = result.AddState(); new_final_states[accept_all_new_state] = true; std::bitset<256> char_set; for (int i = 0; i < result.NumStates(); i++) { char_set.reset(); // Collect all characters that are not accepted by the original FSM. for (const auto& edge : result.GetFsm().GetEdges(i)) { if (edge.IsCharRange()) { for (int j = edge.min; j <= edge.max; ++j) { char_set.set(j); } } } // Add edges for characters that are not accepted. for (int left_bound = 0; left_bound < 256; ++left_bound) { if (char_set[left_bound]) { continue; // Skip characters that are accepted. } int right_bound = left_bound + 1; while (right_bound < 256 && !char_set[right_bound]) { ++right_bound; } result.GetFsm().AddEdge(i, accept_all_new_state, left_bound, right_bound - 1); left_bound = right_bound; } } result.SetEndStates(new_final_states); return ResultOk(result); } FSMWithStartEnd FSMWithStartEnd::Union(const std::vector& fsms) { // Put all the FSMs in parallel. // Allocate a new start state. Start state will be linked to the start states of all the FSMs. // The end states of the new FSM will be the union of the end states of all the FSMs. if (fsms.size() == 1) { return fsms[0]; } XGRAMMAR_DCHECK(fsms.size() > 1) << "Union of 0 FSMs is not allowed."; FSM fsm(1); int start = 0; std::vector ends(1, false); std::vector state_mapping; for (const auto& fsm_with_se : fsms) { fsm.AddFSM(fsm_with_se.GetFsm(), &state_mapping); fsm.AddEpsilonEdge(start, state_mapping[fsm_with_se.GetStart()]); for (int state = 0; state < fsm_with_se.NumStates(); ++state) { ends.push_back(fsm_with_se.IsEndState(state)); } } return FSMWithStartEnd(fsm, start, ends); } FSMWithStartEnd FSMWithStartEnd::Concat(const std::vector& fsms) { // For each FSM, link the end states to the start state of the next FSM. // Set the start state of the first FSM as the start state of the result. // Set the end states of the last FSM as the end states of the result. if (fsms.size() == 1) { return fsms[0]; } XGRAMMAR_DCHECK(fsms.size() > 1) << "Concatenation of 0 FSMs is not allowed."; FSM fsm; int start = 0; std::vector ends; std::vector state_mapping; std::vector previous_ends; for (int i = 0; i < static_cast(fsms.size()); ++i) { fsm.AddFSM(fsms[i].GetFsm(), &state_mapping); if (i == 0) { start = state_mapping[fsms[i].GetStart()]; } else { auto this_start = state_mapping[fsms[i].GetStart()]; for (const auto& end : previous_ends) { fsm.AddEpsilonEdge(end, this_start); } } if (i == static_cast(fsms.size()) - 1) { ends.resize(fsm.NumStates(), false); for (int end = 0; end < fsms[i].NumStates(); ++end) { if (fsms[i].IsEndState(end)) { ends[state_mapping[end]] = true; } } } else { previous_ends.clear(); previous_ends.reserve(fsms[i].GetFsm().NumStates()); for (int end = 0; end < fsms[i].NumStates(); ++end) { if (fsms[i].IsEndState(end)) { previous_ends.push_back(state_mapping[end]); } } } } return FSMWithStartEnd(fsm, start, ends); } Result FSMWithStartEnd::Intersect( const FSMWithStartEnd& lhs, const FSMWithStartEnd& rhs, int max_result_num_states ) { if (!lhs.IsLeaf() || !rhs.IsLeaf()) { return ResultErr("Intersect only support leaf fsm!"); } auto lhs_dfa_raw = lhs.ToDFA(); auto rhs_dfa_raw = rhs.ToDFA(); if (lhs_dfa_raw.IsErr()) { return lhs_dfa_raw; } if (rhs_dfa_raw.IsErr()) { return rhs_dfa_raw; } auto lhs_dfa = std::move(lhs_dfa_raw).Unwrap(); auto rhs_dfa = std::move(rhs_dfa_raw).Unwrap(); // Initialize the result FSM. FSM result_fsm(0); FSMWithStartEnd result(result_fsm, 0, std::vector(), true); std::unordered_map, int> state_map; std::unordered_set> visited; std::queue> queue; queue.push({lhs_dfa.GetStart(), rhs_dfa.GetStart()}); result.AddState(); state_map[{lhs_dfa.GetStart(), rhs_dfa.GetStart()}] = 0; while (!queue.empty()) { auto [lhs_state, rhs_state] = std::move(queue.front()); if (lhs_dfa.IsEndState(lhs_state) && rhs_dfa.IsEndState(rhs_state)) { result.AddEndState(state_map[{lhs_state, rhs_state}]); } queue.pop(); for (const auto& lhs_edge : lhs_dfa.GetFsm().GetEdges(lhs_state)) { for (const auto& rhs_edge : rhs_dfa.GetFsm().GetEdges(rhs_state)) { XGRAMMAR_DCHECK(lhs_edge.IsCharRange() && rhs_edge.IsCharRange()); // Check if the edges intersect. if (lhs_edge.min > rhs_edge.max || rhs_edge.min > lhs_edge.max) { continue; // No intersection. } int min_value = std::max(lhs_edge.min, rhs_edge.min); int max_value = std::min(lhs_edge.max, rhs_edge.max); if (state_map.find(std::make_pair(lhs_edge.target, rhs_edge.target)) == state_map.end()) { state_map[{lhs_edge.target, rhs_edge.target}] = result.AddState(); queue.push({lhs_edge.target, rhs_edge.target}); } int target_state = state_map[{lhs_edge.target, rhs_edge.target}]; result.GetFsm().AddEdge( state_map[{lhs_state, rhs_state}], target_state, min_value, max_value ); } } } return ResultOk(std::move(result)); } bool FSMWithStartEnd::IsDFA() { if (is_dfa_) { return true; } std::bitset<256> character_transitions; std::unordered_set rule_transitions; for (const auto& edges : fsm_->GetEdges()) { character_transitions.reset(); rule_transitions.clear(); for (const auto& edge : edges) { if (edge.IsEpsilon()) { return false; // Epsilon transitions are not allowed in DFA. } if (edge.IsCharRange()) { for (int i = edge.min; i <= edge.max; ++i) { if (character_transitions[i]) { return false; // Duplicate character transition. } character_transitions.set(i); } continue; } if (edge.IsRuleRef()) { if (rule_transitions.find(edge.GetRefRuleId()) != rule_transitions.end()) { return false; // Duplicate rule transition. } rule_transitions.insert(edge.GetRefRuleId()); } // kRepeatRef: by invariant, a state with kRepeatRef has exactly one edge, always // deterministic. } } is_dfa_ = true; return true; } FSMWithStartEnd FSMWithStartEnd::SimplifyEpsilon(int max_num_states) const { if (is_dfa_) { return *this; } if (NumStates() > max_num_states) { return *this; } UnionFindSet union_find_set; std::vector in_degree(NumStates(), 0); std::vector> epsilon_edges; std::vector has_exclude_token(NumStates(), false); for (int i = 0; i < NumStates(); i++) { for (const auto& edge : fsm_->GetEdges(i)) { if (edge.IsExcludeToken()) { has_exclude_token[i] = true; break; } } } for (int i = 0; i < NumStates(); i++) { const auto& edges = fsm_->GetEdges(i); for (const auto& edge : edges) { in_degree[edge.target]++; if (edge.IsEpsilon()) { if (edges.size() == 1 && !has_exclude_token[i] && !has_exclude_token[edge.target]) { // a -- epsilon --> b, and a doesn't have other outward edges. union_find_set.Add(i); union_find_set.Add(edge.target); union_find_set.Union(i, edge.target); in_degree[edge.target]--; // Remove the inward edge since a and b are merged. } else { // a has other outward edges, we store it to check for another case. epsilon_edges.emplace_back(i, edge.target); } } } } // Build the equivalent graph. std::vector equiv_node(NumStates()); for (int i = 0; i < NumStates(); i++) { if (union_find_set.Count(i)) { equiv_node[i] = union_find_set.Find(i); if (equiv_node[i] == i) { continue; } in_degree[equiv_node[i]] += in_degree[i]; } else { equiv_node[i] = i; } } // a --> epsilon --> b, and b doesn't have other inward edges. for (const auto& [from_raw, to_raw] : epsilon_edges) { const int& from = equiv_node[from_raw]; const int& to = equiv_node[to_raw]; if (in_degree[to] == 1 && equiv_node[GetStart()] != to && !has_exclude_token[from_raw] && !has_exclude_token[to_raw]) { union_find_set.Add(from); union_find_set.Add(to); union_find_set.Union(from, to); } } // Merge the states. auto eq_classes = union_find_set.GetAllSets(); if (eq_classes.empty()) { return *this; } std::vector new_to_old(NumStates(), -1); for (size_t i = 0; i < eq_classes.size(); i++) { for (const auto& state : eq_classes[i]) { new_to_old[state] = i; } } int cnt = eq_classes.size(); for (int i = 0; i < NumStates(); i++) { if (new_to_old[i] == -1) { new_to_old[i] = cnt; cnt++; } } return RebuildWithMapping(new_to_old, cnt); } FSMWithStartEnd FSMWithStartEnd::MergeEquivalentStates(int max_result_num_states) const { if (max_result_num_states < NumStates()) { return *this; } // No merge is possible with fewer than 4 states (need >=2 sources sharing a target, // or >=2 sinks sharing a source). if (NumStates() < 4) { return Copy(); } bool changed = true; FSMWithStartEnd result = Copy(); result.GetFsm()->SortEdges(); UnionFindSet union_find_set; // A compact edge view used for incoming/outgoing CSR rows. `peer` means source state in // incoming_edges and target state in outgoing_edges. struct EndpointEdge { int peer; // source in incoming_edges, target in outgoing_edges int32_t min; int32_t max; bool operator<(const EndpointEdge& other) const { return std::tie(peer, min, max) < std::tie(other.peer, other.min, other.max); } }; // Scratch buffers reused across iterations to avoid repeated vector allocation. // Number of incoming edges for each state, used to size the incoming CSR rows. std::vector incoming_row_sizes; // Number of outgoing edges for each state, used to size the outgoing CSR rows. std::vector outgoing_row_sizes; // Write positions while filling incoming_edges rows. std::vector incoming_write_positions; // Write positions while filling outgoing_edges rows. std::vector outgoing_write_positions; // Incoming edges grouped by target state. Compact2DArray incoming_edges; // Outgoing edges grouped by source state. Compact2DArray outgoing_edges; // Number of distinct predecessor states for each state. std::vector incoming_distinct_count; // Number of distinct successor states for each state. std::vector outgoing_distinct_count; // The only predecessor state when incoming_distinct_count[state] == 1. std::vector single_incoming_source; // The only successor state when outgoing_distinct_count[state] == 1. std::vector single_outgoing_target; // Terminal end states collected for leaf-state merging. std::vector no_successor_end_states; // Terminal non-end states collected for leaf-state merging. std::vector no_successor_non_end_states; while (changed) { int n = result.NumStates(); union_find_set.Clear(); // First pass: count row sizes for the incoming/outgoing CSR arrays. incoming_row_sizes.assign(n, 0); outgoing_row_sizes.assign(n, 0); for (int source = 0; source < n; ++source) { const auto& edges = result.GetFsm().GetEdges(source); outgoing_row_sizes[source] = static_cast(edges.size()); for (const auto& edge : edges) { ++incoming_row_sizes[edge.target]; } } // Allocate CSR rows. The underlying storage is reset and reused across iterations. incoming_edges.ResetWithRowSizes(incoming_row_sizes); outgoing_edges.ResetWithRowSizes(outgoing_row_sizes); incoming_write_positions.assign(n, 0); outgoing_write_positions.assign(n, 0); // Second pass: fill incoming and outgoing rows. Incoming rows are naturally grouped by // source because we scan source states in order; outgoing rows are sorted by target below. for (int source = 0; source < n; ++source) { const auto& edges = result.GetFsm().GetEdges(source); auto outgoing_row = outgoing_edges.MutableRowAt(source); for (const auto& edge : edges) { incoming_edges.MutableRowAt(edge.target )[incoming_write_positions[edge.target]++] = {source, edge.min, edge.max}; outgoing_row[outgoing_write_positions[source]++] = {edge.target, edge.min, edge.max}; } std::sort(outgoing_row.begin(), outgoing_row.end()); } // Identify states with exactly one distinct predecessor/successor. These states are the // candidates for the two local merge rules below. incoming_distinct_count.assign(n, 0); outgoing_distinct_count.assign(n, 0); single_incoming_source.assign(n, -1); single_outgoing_target.assign(n, -1); for (int state = 0; state < n; ++state) { auto incoming_row = incoming_edges[state]; if (incoming_row.size() > 0) { incoming_distinct_count[state] = 1; single_incoming_source[state] = incoming_row[0].peer; for (int32_t i = 1; i < incoming_row.size(); ++i) { if (incoming_row[i].peer != incoming_row[i - 1].peer) { ++incoming_distinct_count[state]; single_incoming_source[state] = -1; } } } auto outgoing_row = outgoing_edges[state]; if (outgoing_row.size() > 0) { outgoing_distinct_count[state] = 1; single_outgoing_target[state] = outgoing_row[0].peer; for (int32_t i = 1; i < outgoing_row.size(); ++i) { if (outgoing_row[i].peer != outgoing_row[i - 1].peer) { ++outgoing_distinct_count[state]; single_outgoing_target[state] = -1; } } } } // Case 1: Like ab | ac | ad, then they can be merged into a(b | c | d). bool is_equiv_successor = false; for (int i = 0; i < n; i++) { if (incoming_distinct_count[i] != 1 || union_find_set.Count(i)) { continue; } int previous_state = single_incoming_source[i]; auto edges_to_i = incoming_edges[i]; auto siblings = outgoing_edges[previous_state]; int32_t group_begin = 0; while (group_begin < siblings.size()) { int sibling = siblings[group_begin].peer; int32_t group_end = group_begin + 1; while (group_end < siblings.size() && siblings[group_end].peer == sibling) { ++group_end; } auto edges_to_sibling = siblings.Slice(group_begin, group_end); group_begin = group_end; if (sibling <= i || incoming_distinct_count[sibling] != 1 || result.IsEndState(sibling) != result.IsEndState(i)) { continue; } // Check if the edges from previous_state to i and sibling are the same. if (edges_to_i.size() != edges_to_sibling.size()) { continue; // Different edges, not equivalent. } bool is_equiv = true; for (int32_t j = 0; j < edges_to_i.size(); ++j) { if (edges_to_i[j].min != edges_to_sibling[j].min || edges_to_i[j].max != edges_to_sibling[j].max) { is_equiv = false; break; // Different edge ranges, not equivalent. } } // Merge the equivalent successor states. if (is_equiv) { union_find_set.Add(i); union_find_set.Add(sibling); union_find_set.Union(i, sibling); is_equiv_successor = true; } } } // Case 2: Like ba | ca | da, then they can be merged into (b | c | d)a. bool is_equiv_precursor = false; no_successor_end_states.clear(); no_successor_non_end_states.clear(); for (int i = 0; i < n; i++) { int outgoing_count = outgoing_distinct_count[i]; if (outgoing_count == 0) { if (result.IsEndState(i)) { no_successor_end_states.push_back(i); } else { no_successor_non_end_states.push_back(i); } continue; // Skip states with no successors. } if (outgoing_count != 1 || union_find_set.Count(i)) { continue; // Skip states with multiple successors. } int next_state = single_outgoing_target[i]; auto node_edges = outgoing_edges[i]; auto siblings = incoming_edges[next_state]; int32_t group_begin = 0; while (group_begin < siblings.size()) { int sibling = siblings[group_begin].peer; while (group_begin < siblings.size() && siblings[group_begin].peer == sibling) { ++group_begin; } // Avoid chaining a Case 2 merge onto a state already merged earlier in this iteration // (typically by Case 1), which can over-merge via transitive closure. if (sibling <= i || union_find_set.Count(sibling) || outgoing_distinct_count[sibling] != 1 || result.IsEndState(i) != result.IsEndState(sibling)) { continue; } auto sibling_node_edges = outgoing_edges[sibling]; if (sibling_node_edges.size() != node_edges.size()) { continue; // Different number of edges, not equivalent. } // Check if the sibling state has the same outgoing edges as i. bool is_equiv = true; for (int32_t j = 0; j < node_edges.size(); ++j) { if (sibling_node_edges[j].min != node_edges[j].min || sibling_node_edges[j].max != node_edges[j].max) { is_equiv = false; break; } } // Merge the equivalent precursor states. if (is_equiv) { union_find_set.Add(i); union_find_set.Add(sibling); union_find_set.Union(i, sibling); is_equiv_precursor = true; } } } if (no_successor_end_states.size() > 1) { // Merge all end states with no successors. for (size_t i = 1; i < no_successor_end_states.size(); ++i) { union_find_set.Add(no_successor_end_states[0]); union_find_set.Add(no_successor_end_states[i]); union_find_set.Union(no_successor_end_states[0], no_successor_end_states[i]); is_equiv_precursor = true; } } if (no_successor_non_end_states.size() > 1) { // Merge all non-end states with no successors. for (size_t i = 1; i < no_successor_non_end_states.size(); ++i) { union_find_set.Add(no_successor_non_end_states[0]); union_find_set.Add(no_successor_non_end_states[i]); union_find_set.Union(no_successor_non_end_states[0], no_successor_non_end_states[i]); is_equiv_precursor = true; } } changed = is_equiv_successor || is_equiv_precursor; if (changed) { // Rebuild the FSM with the equivalent states merged, then repeat until no local merge // rule applies. auto eq_classes = union_find_set.GetAllSets(); std::vector old_to_new(result.NumStates(), -1); for (size_t i = 0; i < eq_classes.size(); i++) { for (const auto& state : eq_classes[i]) { old_to_new[state] = i; } } int cnt = eq_classes.size(); for (int i = 0; i < result.NumStates(); i++) { if (old_to_new[i] == -1) { old_to_new[i] = cnt; cnt++; } } result = result.RebuildWithMapping(old_to_new, cnt); result.GetFsm()->SortEdges(); } } return result; } Result FSMWithStartEnd::MinimizeDFA(int max_num_states) const { FSMWithStartEnd now_fsm(FSM(0), 0, std::vector(), true); if (NumStates() > max_num_states) { return ResultErr("The number of states exceeds the limit."); } // To perform the algorithm, we must make sure the FSM is // a DFA. if (!is_dfa_) { Result dfa_raw = ToDFA(max_num_states); if (dfa_raw.IsErr()) { return dfa_raw; } now_fsm = std::move(dfa_raw).Unwrap(); } else { now_fsm = Copy(); } // Initialize the precursors of nodes. std::vector, int>>> precursors; precursors.resize(now_fsm.NumStates()); for (int i = 0; i < now_fsm.NumStates(); ++i) { const auto& edges = now_fsm.GetFsm().GetEdges(i); for (const auto& edge : edges) { XGRAMMAR_DCHECK(!edge.IsEpsilon()); precursors[edge.target].push_back(std::make_pair(std::make_pair(edge.min, edge.max), i)); } } // Initialize the partitions and working set. std::vector> partitions; std::vector> working_set; std::unordered_set final_states; std::unordered_set non_final_states; for (int i = 0; i < now_fsm.NumStates(); ++i) { if (now_fsm.IsEndState(i)) { final_states.insert(i); } else { non_final_states.insert(i); } } partitions.push_back(final_states); partitions.push_back(non_final_states); working_set.push_back(std::move(final_states)); working_set.push_back(std::move(non_final_states)); while (!working_set.empty()) { std::map, std::unordered_set> possible_transitions; auto current_partition = std::move(working_set.back()); working_set.pop_back(); // Get the possible transitions from the current partition. for (const auto& state : current_partition) { const auto& precursor_map = precursors[state]; for (const auto& precursor : precursor_map) { if (possible_transitions.find(precursor.first) == possible_transitions.end()) { possible_transitions[precursor.first] = std::unordered_set(); } possible_transitions[precursor.first].insert(precursor.second); } } // Check each possible transition. std::vector intersection; std::vector difference; for (const auto& [transition, precursors] : possible_transitions) { for (size_t i = 0; i < partitions.size(); i++) { const auto& partition = partitions[i]; intersection.clear(); // partition \cap precursors difference.clear(); // partition - precursors for (const auto& partition_state : partition) { if (precursors.find(partition_state) != precursors.end()) { intersection.push_back(partition_state); } else { difference.push_back(partition_state); } } // the states in the partition is not equivalent. We need to // update the working set and the partitions. if ((!intersection.empty()) && (!difference.empty())) { bool in_working_set = false; for (size_t i = 0; i < working_set.size(); i++) { if (partition == working_set[i]) { in_working_set = true; working_set[i].clear(); for (const auto& state : intersection) { working_set[i].insert(state); } working_set.emplace_back(); for (const auto& state : difference) { working_set.back().insert(state); } break; } } if (!in_working_set) { const auto& smaller_set = difference.size() < intersection.size() ? difference : intersection; working_set.emplace_back(); for (const auto& state : smaller_set) { working_set.back().insert(state); } } partitions[i].clear(); for (const auto& state : intersection) { partitions[i].insert(state); } partitions.emplace_back(); for (const auto& state : difference) { partitions.back().insert(state); } } } } } std::vector state_mapping(now_fsm.NumStates(), -1); for (size_t i = 0; i < partitions.size(); ++i) { for (const auto& state : partitions[i]) { state_mapping[state] = i; } } int new_num_states = partitions.size(); return ResultOk(now_fsm.RebuildWithMapping(state_mapping, new_num_states)); } Result FSMWithStartEnd::ToDFA(int max_num_states) const { if (NumStates() > max_num_states) { return ResultErr("The number of states exceeds the limit."); } FSMWithStartEnd dfa(FSM(0), 0, std::vector(), true); std::vector> closures; std::unordered_set rules; std::unordered_set repeat_aux_indices; int now_process = 0; std::unordered_set closure; closure.insert(start_); fsm_.GetEpsilonClosure(&closure); closures.push_back(closure); while (now_process < static_cast(closures.size())) { rules.clear(); repeat_aux_indices.clear(); std::unordered_set token_aux_indices; std::unordered_set exclude_token_aux_indices; std::set interval_ends; std::bitset<256> allowed_characters; dfa.AddState(); // Check if the closure is a final state. for (const auto& state : closures[now_process]) { if (IsEndState(state)) { dfa.AddEndState(now_process); } const auto& edges = fsm_->GetEdges(state); for (const auto& edge : edges) { if (edge.IsCharRange()) { interval_ends.insert(edge.min); interval_ends.insert(edge.max + 1); for (int i = edge.min; i <= edge.max; ++i) { allowed_characters.set(i); } continue; } else if (edge.IsRuleRef()) { rules.insert(edge.GetRefRuleId()); } else if (edge.IsRepeatRef()) { repeat_aux_indices.insert(edge.GetAuxIndex()); } else if (edge.IsToken()) { token_aux_indices.insert(edge.GetAuxIndex()); } else if (edge.IsExcludeToken()) { exclude_token_aux_indices.insert(edge.GetAuxIndex()); } } } // This part is to get the all possible intervals. // Which can help reduce the transitions. using Interval = std::pair; std::vector intervals; intervals.reserve(interval_ends.size()); int last = -1; for (const auto& end : interval_ends) { if (last == -1) { last = end; continue; } bool allowed = true; for (int i = last; i < end; ++i) { if (!allowed_characters[i]) { allowed = false; break; } } if (allowed) { intervals.emplace_back(last, end - 1); } last = end; } for (const auto& interval : intervals) { std::unordered_set next_closure; for (const auto& state : closures[now_process]) { const auto& edges = fsm_->GetEdges(state); for (const auto& edge : edges) { if (edge.IsCharRange()) { if (interval.first >= edge.min && interval.second <= edge.max) { if (next_closure.find(edge.target) == next_closure.end()) { std::unordered_set epsilon_closure; epsilon_closure.insert(edge.target); fsm_.GetEpsilonClosure(&epsilon_closure); next_closure.insert(epsilon_closure.begin(), epsilon_closure.end()); } } } } } bool flag = false; for (int j = 0; j < static_cast(closures.size()); j++) { if (closures[j] == next_closure) { dfa.GetFsm().AddEdge(now_process, j, interval.first, interval.second); flag = true; break; } } if (!flag) { dfa.GetFsm().AddEdge(now_process, closures.size(), interval.first, interval.second); closures.push_back(next_closure); } } for (auto rule : rules) { std::unordered_set next_closure; for (const auto& state : closures[now_process]) { const auto& edges = fsm_.GetEdges(state); for (const auto& edge : edges) { if (edge.IsRuleRef()) { if (rule == edge.GetRefRuleId()) { if (next_closure.find(edge.target) == next_closure.end()) { std::unordered_set epsilon_closure; epsilon_closure.insert(edge.target); fsm_.GetEpsilonClosure(&epsilon_closure); next_closure.insert(epsilon_closure.begin(), epsilon_closure.end()); } } } } } bool flag = false; for (int j = 0; j < static_cast(closures.size()); j++) { if (closures[j] == next_closure) { dfa.GetFsm().AddRuleEdge(now_process, j, rule); flag = true; break; } } if (!flag) { dfa.GetFsm().AddRuleEdge(now_process, closures.size(), rule); closures.push_back(next_closure); } } for (auto aux_idx : repeat_aux_indices) { std::unordered_set next_closure; for (const auto& state : closures[now_process]) { const auto& edges = fsm_.GetEdges(state); for (const auto& edge : edges) { if (edge.IsRepeatRef() && edge.GetAuxIndex() == aux_idx) { if (next_closure.find(edge.target) == next_closure.end()) { std::unordered_set epsilon_closure; epsilon_closure.insert(edge.target); fsm_.GetEpsilonClosure(&epsilon_closure); next_closure.insert(epsilon_closure.begin(), epsilon_closure.end()); } } } } bool flag = false; for (int j = 0; j < static_cast(closures.size()); j++) { if (closures[j] == next_closure) { dfa.GetFsm().AddEdge(now_process, j, FSMEdge::EdgeType::kRepeatRef, aux_idx); flag = true; break; } } if (!flag) { dfa.GetFsm().AddEdge(now_process, closures.size(), FSMEdge::EdgeType::kRepeatRef, aux_idx); closures.push_back(next_closure); } } for (auto aux_idx : token_aux_indices) { std::unordered_set next_closure; for (const auto& state : closures[now_process]) { const auto& edges = fsm_.GetEdges(state); for (const auto& edge : edges) { if (edge.IsToken() && edge.GetAuxIndex() == aux_idx) { if (next_closure.find(edge.target) == next_closure.end()) { std::unordered_set epsilon_closure; epsilon_closure.insert(edge.target); fsm_.GetEpsilonClosure(&epsilon_closure); next_closure.insert(epsilon_closure.begin(), epsilon_closure.end()); } } } } bool flag = false; for (int j = 0; j < static_cast(closures.size()); j++) { if (closures[j] == next_closure) { dfa.GetFsm().AddEdge(now_process, j, FSMEdge::EdgeType::kToken, aux_idx); flag = true; break; } } if (!flag) { dfa.GetFsm().AddEdge(now_process, closures.size(), FSMEdge::EdgeType::kToken, aux_idx); closures.push_back(next_closure); } } for (auto aux_idx : exclude_token_aux_indices) { std::unordered_set next_closure; for (const auto& state : closures[now_process]) { const auto& edges = fsm_.GetEdges(state); for (const auto& edge : edges) { if (edge.IsExcludeToken() && edge.GetAuxIndex() == aux_idx) { if (next_closure.find(edge.target) == next_closure.end()) { std::unordered_set epsilon_closure; epsilon_closure.insert(edge.target); fsm_.GetEpsilonClosure(&epsilon_closure); next_closure.insert(epsilon_closure.begin(), epsilon_closure.end()); } } } } bool flag = false; for (int j = 0; j < static_cast(closures.size()); j++) { if (closures[j] == next_closure) { dfa.GetFsm().AddEdge(now_process, j, FSMEdge::EdgeType::kExcludeToken, aux_idx); flag = true; break; } } if (!flag) { dfa.GetFsm().AddEdge( now_process, closures.size(), FSMEdge::EdgeType::kExcludeToken, aux_idx ); closures.push_back(next_closure); } } now_process++; } dfa.GetFsm().SetEdgeAuxData(std::vector(fsm_.GetEdgeAuxData())); dfa.is_dfa_ = true; return ResultOk(dfa); } /****************** CompactFSMWithStartEnd ******************/ std::string CompactFSMWithStartEnd::ToString() const { std::string result; result += "CompactFSM(num_states=" + std::to_string(NumStates()) + ", start=" + std::to_string(start_) + ", end=["; std::unordered_set reachable_states; GetReachableStates(&reachable_states); std::vector reachable_states_vec(reachable_states.begin(), reachable_states.end()); std::sort(reachable_states_vec.begin(), reachable_states_vec.end()); bool first = true; for (int end = 0; end < NumStates(); end++) { if (reachable_states.count(end) && IsEndState(end)) { if (!first) { result += ", "; } first = false; result += std::to_string(end); } } result += "], edges=" + fsm_.EdgesToString(reachable_states_vec) + ")"; return result; } std::ostream& operator<<(std::ostream& os, const CompactFSMWithStartEnd& fsm) { os << fsm.ToString(); return os; } std::size_t MemorySize(const CompactFSM& self) { return MemorySize(*self.ImplPtr()); } std::size_t MemorySize(const CompactFSMWithStartEnd& self) { return MemorySize(self.fsm_) + MemorySize(self.ends_); } FSMWithStartEnd CompactFSMWithStartEnd::ToFSM() const { return FSMWithStartEnd(fsm_.ToFSM(), start_, ends_); } } // namespace xgrammar xgrammar-0.2.3/cpp/fsm.h000066400000000000000000001003101521764210300150460ustar00rootroot00000000000000/*! * Copyright (c) 2025 by Contributors * \file xgrammar/fsm.h * \note For functions accepting a pointer to a container as result, the container will be cleared * before the result is stored. */ #ifndef XGRAMMAR_FSM_H_ #define XGRAMMAR_FSM_H_ #include #include #include #include #include #include #include #include #include #include #include "support/compact_2d_array.h" #include "support/logging.h" #include "support/reflection.h" #include "support/utils.h" #include "xgrammar/exception.h" namespace xgrammar { /*! * \brief The edge of a FSM. */ struct FSMEdge { /*! * \brief Edge type is encoded in the `min` field. When min >= 0, the edge is a character range * [min, max]. When min < 0, it is a special edge type identified by the enum values below. * * For each type, `max` has a type-specific meaning (see comments on each enumerator). */ enum EdgeType : int32_t { //! Character range [min, max]. min >= 0. kCharRange = 0, //! Epsilon transition. max is unused. kEpsilon = -1, //! Rule reference. max = rule_id. kRuleRef = -2, //! Accepts the EOS token. max is unused. kEOS = -3, //! Repeated rule reference. max = aux index into edge_aux_data //! (layout: [rule_id, lower, upper]). //! Invariant: a state with a kRepeatRef edge has exactly one outgoing edge. kRepeatRef = -4, //! Accepts a set of token IDs. max = aux index into edge_aux_data //! (layout: [count, token_id_0, token_id_1, ...]). kToken = -5, //! Accepts any token NOT in the given set. max = aux index into edge_aux_data //! (layout: [count, token_id_0, token_id_1, ...]). kExcludeToken = -6, }; inline static constexpr int kMaxChar = 255; int32_t min, max; /*! * \brief The target state id of the edge. */ int32_t target; // for serialization only FSMEdge() = default; FSMEdge(int32_t min, int32_t max, int32_t target) : min(min), max(max), target(target) { XGRAMMAR_DCHECK(!IsCharRange() || min <= max) << "Invalid FSMEdge: min > max. min=" << min << ", max=" << max; } /*! * \brief Compare the edges. Used to sort the edges in the FSM. */ // TODO(yixin): consider combining the fields to a single int64_t for better efficiency friend bool operator==(const FSMEdge& lhs, const FSMEdge& rhs) { return std::make_tuple(lhs.min, lhs.max, lhs.target) == std::make_tuple(rhs.min, rhs.max, rhs.target); } /*! * \brief Compare the edges. Used to sort the edges in the FSM. */ friend bool operator<(const FSMEdge& lhs, const FSMEdge& rhs) { return std::make_tuple(lhs.min, lhs.max, lhs.target) < std::make_tuple(rhs.min, rhs.max, rhs.target); } /*! * \brief Check if the edge is a character range. */ bool IsCharRange() const { return min >= 0; } /*! * \brief Check if the edge is an epsilon transition. */ bool IsEpsilon() const { return min == EdgeType::kEpsilon; } /*! * \brief Check if the edge is a rule reference. */ bool IsRuleRef() const { return min == EdgeType::kRuleRef; } /*! * \brief Check if the edge is an EOS transition. */ bool IsEOS() const { return min == EdgeType::kEOS; } /*! * \brief Check if the edge is a repeat reference. */ bool IsRepeatRef() const { return min == EdgeType::kRepeatRef; } bool IsToken() const { return min == EdgeType::kToken; } bool IsExcludeToken() const { return min == EdgeType::kExcludeToken; } /*! * \brief Get the rule id of the edge. * \return The rule id of the edge. -1 if the edge is not a rule reference. */ int32_t GetRefRuleId() const { return IsRuleRef() ? max : -1; } /*! * \brief Get the auxiliary data index for repeat reference edges. * \return The index into the owning FSM's edge_aux_data. -1 if not a repeat reference. */ int32_t GetAuxIndex() const { return (IsRepeatRef() || IsToken() || IsExcludeToken()) ? max : -1; } /*! \brief Check if the edge uses auxiliary data. */ bool IsAuxEdge() const { return IsRepeatRef() || IsToken() || IsExcludeToken(); } friend struct member_trait; }; /*! \brief View into edge_aux_data for a repeat edge (layout: [rule_id, lower, upper]). */ struct RepeatEdgeRef { const int32_t* data; int32_t RuleId() const { return data[0]; } int32_t Lower() const { return data[1]; } int32_t Upper() const { return data[2]; } }; /*! \brief View into edge_aux_data for a token edge (layout: [count, token_id_0, ...]). */ struct TokenEdgeRef { const int32_t* data; int32_t Count() const { return data[0]; } const int32_t* TokenIds() const { return data + 1; } bool Contains(int32_t token_id) const { for (int32_t i = 0; i < Count(); ++i) { if (TokenIds()[i] == token_id) return true; } return false; } }; /*! \brief View into edge_aux_data for an exclude-token edge (layout: [count, token_id_0, ...]). */ struct ExcludeTokenEdgeRef { const int32_t* data; int32_t Count() const { return data[0]; } const int32_t* TokenIds() const { return data + 1; } bool Contains(int32_t token_id) const { for (int32_t i = 0; i < Count(); ++i) { if (TokenIds()[i] == token_id) return true; } return false; } bool Accepts(int32_t token_id) const { return !Contains(token_id); } }; /*! * \brief Comparator for FSMEdge. Only compare the min and max. */ struct FSMEdgeRangeComparator { bool operator()(const FSMEdge& lhs, const FSMEdge& rhs) const { return std::make_tuple(lhs.min, lhs.max) < std::make_tuple(rhs.min, rhs.max); } }; XGRAMMAR_MEMBER_ARRAY(FSMEdge, &FSMEdge::min, &FSMEdge::max, &FSMEdge::target); } // namespace xgrammar XGRAMMAR_HASH_BY_MEMBERS( xgrammar::FSMEdge, &xgrammar::FSMEdge::min, &xgrammar::FSMEdge::max, &xgrammar::FSMEdge::target ); namespace xgrammar { class CompactFSM; /*! * \brief FSM is a class that represents a finite state machine, could be a DFA or an NFA. * \details It's mutable, which means you can add edges and states to it. */ class FSM { public: /*! * \brief Construct an FSM with a given number of states. * \param num_states The number of states in the FSM. */ FSM(int num_states = 0); /*! * \brief Construct an FSM with a given set of edges. */ FSM(const std::vector>& edges, std::vector edge_aux_data = {}); /*! * \brief Construct an FSM with a given set of edges. */ FSM(std::vector>&& edges, std::vector edge_aux_data = {}); /****************** FSM Visitors ******************/ /*! * \brief Get the number of states in the FSM. * \return The number of states in the FSM. */ int NumStates() const; /*! * \brief Get the edges of the FSM. * \return The edges of the FSM. */ const std::vector>& GetEdges() const; /*! * \brief Get the edges of the FSM. * \return The edges of the FSM. */ std::vector>& GetEdges(); /*! * \brief Get the edges of the FSM. * \param state The state to get the edges from. * \return The edges of the FSM. */ std::vector& GetEdges(int state); /*! * \brief Get the edges of the FSM. * \param state The state to get the edges from. * \return The edges of the FSM. */ const std::vector& GetEdges(int state) const; /*! * \brief Convert the edges of the FSM to a string. Used in printing the FSM. * \return The string representation of the edges of the FSM. */ std::string EdgesToString(std::optional> states = std::nullopt) const; /****************** FSM Traversal Visitors ******************/ inline static constexpr int kNoNextState = -1; /*! * \brief Advance the FSM from a given state based on an input character. If there are multiple * transitions, the first one will be returned. * \param from The source state to transition from. * \param character The input character. * \return The target state if a valid transition exists, kNoNextState otherwise. */ int GetNextState(int from, int value, FSMEdge::EdgeType edge_type = FSMEdge::EdgeType::kCharRange) const; /*! * \brief Advance the FSM to the next state. * \param from The current states. * \param value The input value. * \param result The possible next states. The result is cleared at the beginning. * \param value_is_rule Whether the input value is a rule id. * \param from_is_closure Whether from is an epsilon closure. */ void Advance( const std::unordered_set& from, int value, std::unordered_set* result, FSMEdge::EdgeType edge_type = FSMEdge::EdgeType::kCharRange, bool from_is_closure = false ) const; /*! * \brief Get all the possible rule numbers for a given state. * \param state_num The state number. * \param rules The set of possible rule numbers. The result is cleared at the beginning. */ void GetPossibleRules(int state_num, std::unordered_set* rules) const; /*! * \brief Get the epsilon closure of a set of states, i.e. those can be reached by epsilon * transitions. * \param state_set The states in the epsilon closure. The result is not cleared. */ void GetEpsilonClosure(std::unordered_set* state_set) const; /*! * \brief Get the reachable states from a set of states. * \param from The current states. * \param result The reachable states. The result is cleared at the beginning. */ void GetReachableStates(const std::vector& from, std::unordered_set* result) const; /****************** FSM Mutators ******************/ /*! * \brief Adds a new state to the FSM. * \return The index of the newly added state. */ int AddState(); /*! * \brief Adds a transition edge between states with given min and max values. For character * transitions, it accepts any character in range [min, max]. * \param from The source state. * \param to The target state. * \param min The min value of the range. * \param max The max value of the range. */ void AddEdge(int from, int to, int32_t min, int32_t max); /*! \brief Add a raw edge with explicit type and value. */ void AddEdge(int from, int to, FSMEdge::EdgeType type, int32_t value); /*! * \brief Add an epsilon transition between two states. * \param from The source state. * \param to The target state. */ void AddEpsilonEdge(int from, int to); /*! * \brief Add a rule reference edge between states. * \param from The source state. * \param to The target state. * \param rule_id The rule id to reference. */ void AddRuleEdge(int from, int to, int32_t rule_id); /*! * \brief Add an EOS transition between two states. * \param from The source state. * \param to The target state. */ void AddEOSEdge(int from, int to); /*! * \brief Add a repeat reference edge between states, allocating auxiliary data. * \param from The source state. * \param to The target state. * \param rule_id The rule to repeat. * \param lower Minimum repeat count. * \param upper Maximum repeat count (-1 for unlimited). */ void AddRepeatEdge(int from, int to, int32_t rule_id, int32_t lower, int32_t upper); void AddTokenEdge(int from, int to, const std::vector& token_ids); void AddExcludeTokenEdge(int from, int to, const std::vector& token_ids); /*! \brief Get the edge auxiliary data. */ const std::vector& GetEdgeAuxData() const; /*! \brief Set the edge auxiliary data (used during FSM construction). */ void SetEdgeAuxData(std::vector data); /*! \brief Get repeat edge info by aux index. */ RepeatEdgeRef GetRepeatEdgeInfo(int32_t idx) const; /*! \brief Get token edge info by aux index. */ TokenEdgeRef GetTokenEdgeInfo(int32_t idx) const; /*! \brief Get exclude-token edge info by aux index. */ ExcludeTokenEdgeRef GetExcludeTokenEdgeInfo(int32_t idx) const; /*! * \brief Add a whole FSM to the current FSM. * \param fsm The FSM to be added. * \param state_mapping The mapping from the state ids of the added FSM to the new ids in the * current FSM. The result is cleared at the beginning. If the fsm's state id starts from 0, use * it for efficiency. */ void AddFSM(const FSM& fsm, std::vector* state_mapping = nullptr); /****************** FSM Construction Methods ******************/ /*! \brief Return a copy of the FSM. */ FSM Copy() const; /*! * \brief Rebuild the FSM with the new state ids. * \param state_mapping The mapping from the old state ids to the new state ids. * \param new_num_states The new number of states. * \return The rebuilt FSM. */ FSM RebuildWithMapping(const std::vector& state_mapping, int new_num_states) const; /*! * \brief Sort the edges of the FSM by their min, max and target. */ void SortEdges(); /*! * \brief Transform a FSM to a compact FSM. This method will first sort the edges of the FSM, * then put all the edges into a compact array. * \return The compact FSM. */ CompactFSM ToCompact(); XGRAMMAR_DEFINE_PIMPL_METHODS(FSM); }; /*! * \brief CompactFSM is the compact from of FSM. * \details It uses Compact2DArray to store the edges, ensuring memory contiguity. It sorts all * outgoing edges from a node according to their min and max values, so traversal can be faster. * * CompactFSM is immutable. If you need to modify a CompactFSM, you need to convert it to a FSM * first, and convert it back after modification. * * It share the same set of visitor methods with FSM. */ class CompactFSM { public: // for serialization only CompactFSM() = default; explicit CompactFSM( const Compact2DArray& edges, std::vector edge_aux_data = {} ); explicit CompactFSM(Compact2DArray&& edges, std::vector edge_aux_data = {}); /****************** CompactFSM Visitors ******************/ /*! * \brief Get the number of states in the FSM. * \return The number of states in the FSM. */ int NumStates() const; /*! * \brief Get the edges of the CompactFSM. * \return The edges of the CompactFSM. */ const Compact2DArray& GetEdges() const; /*! * \brief Get the edges of the CompactFSM. * \param state The state to get the edges from. * \return The edges of the CompactFSM. */ Compact2DArray::Row GetEdges(int state) const; /*! * \brief Convert the edges of the CompactFSM to a string. Used in printing the CompactFSM. * \return The string representation of the edges of the CompactFSM. */ std::string EdgesToString(std::optional> states = std::nullopt) const; /*! * \brief Get the memory size of the CompactFSM. * \param self The CompactFSM. * \return The memory size of the CompactFSM. */ friend std::size_t MemorySize(const CompactFSM& self); /****************** CompactFSM Traversal Visitors ******************/ inline static constexpr int kNoNextState = -1; /*! * \brief Advance the FSM from a given state based on an input character. If there are multiple * transitions, the first one will be returned. * \param from The source state to transition from. * \param character The input character. * \param targets The target states to be filled with the possible next states. * \return The target state if a valid transition exists, kNoNextState otherwise. */ void GetNextStates( int from, int value, FSMEdge::EdgeType edge_type = FSMEdge::EdgeType::kCharRange, std::vector* targets = nullptr ) const; /*! * \brief Advance the FSM to the next state. * \param from The current states. * \param value The input value. * \param result The possible next states. The result is cleared at the beginning. * \param value_is_rule Whether the input value is a rule id. * \param from_is_closure Whether from is an epsilon closure. */ void Advance( const std::unordered_set& from, int value, std::unordered_set* result, FSMEdge::EdgeType edge_type = FSMEdge::EdgeType::kCharRange, bool from_is_closure = false ) const; /*! * \brief Get all the possible rule numbers for a given state. * \param state_num The state number. * \param rules The set of possible rule numbers. The result is cleared at the beginning. */ void GetPossibleRules(int state_num, std::unordered_set* rules) const; /*! * \brief Get the epsilon closure of a set of states, i.e. those can be reached by epsilon * transitions. * \param state_set The states in the epsilon closure. The result is not cleared. */ void GetEpsilonClosure(std::unordered_set* state_set) const; /*! * \brief Get the reachable states from a set of states. * \param from The current states. * \param result The reachable states. The result is cleared at the beginning. */ void GetReachableStates(const std::vector& from, std::unordered_set* result) const; /*! * \brief Get the number of edges in the compact FSM. * \return The number of edges. */ size_t GetNumEdges() const; /****************** CompactFSM Auxiliary Data ******************/ /*! \brief Get the edge auxiliary data. */ const std::vector& GetEdgeAuxData() const; /*! \brief Set the edge auxiliary data (used during FSM construction). */ void SetEdgeAuxData(std::vector data); /*! \brief Get repeat edge info by aux index. */ RepeatEdgeRef GetRepeatEdgeInfo(int32_t idx) const; /*! \brief Get token edge info by aux index. */ TokenEdgeRef GetTokenEdgeInfo(int32_t idx) const; /*! \brief Get exclude-token edge info by aux index. */ ExcludeTokenEdgeRef GetExcludeTokenEdgeInfo(int32_t idx) const; /****************** CompactFSM Construction Methods ******************/ /*! * \brief Transform the compact FSM to a FSM. * \return The FSM. */ FSM ToFSM() const; friend picojson::value SerializeJSONValue(const CompactFSM& value); friend std::optional DeserializeJSONValue( CompactFSM* result, const picojson::value& value, const std::string& type_name ); XGRAMMAR_DEFINE_PIMPL_METHODS(CompactFSM); }; std::optional DeserializeJSONValue( CompactFSM* result, const picojson::value& value, const std::string& type_name = "" ); class FSMWithStartEnd; class FSMWithStartEndWithSize; class CompactFSMWithStartEnd; class CompactFSMWithStartEndWithSize; struct CompactFSMWithStartEndWithSizeSerializeHelper; /*! * \brief The base class for FSMWithStartEnd and CompactFSMWithStartEnd. It defines the * common constructor and visitor methods. */ template class FSMWithStartEndBase { static_assert( std::is_same_v || std::is_same_v, "FSMType must be FSM or CompactFSM" ); public: // For serialization only FSMWithStartEndBase() = default; FSMWithStartEndBase( const FSMType& fsm, int start, const std::vector& ends, bool is_dfa = false ) : fsm_(fsm), start_(start), ends_(ends), is_dfa_(is_dfa) {} /****************** Member Accessors and Mutators ******************/ /*! \brief Returns the underlying FSM. */ const FSMType& GetFsm() const { return fsm_; } /*! \brief Returns the start state of the FSM. */ int GetStart() const { return start_; } /*! \brief Returns the end states of the FSM. */ const std::vector& GetEnds() const { return ends_; } /*! * \brief Checks if a given state is an end/accepting state. * \param state The state to check. * \return True if the state is an end state, false otherwise. */ bool IsEndState(int state) const { return ends_[state]; } /*! \brief Check if a state is scanable. * \param state The state to check. * \return True if the state is scanable, false otherwise. */ bool IsScanableState(int state) const { for (const auto& edge : fsm_.GetEdges(state)) { if (edge.IsCharRange() || edge.IsToken() || edge.IsExcludeToken()) { return true; } } return false; } /*! * \brief Check if a state is not terminal. * \param state The state to check. * \return True if the state is scanable, false otherwise. */ bool IsNonTerminalState(int state) const { for (const auto& edge : fsm_.GetEdges(state)) { if (edge.IsRuleRef() || edge.IsEpsilon() || edge.IsRepeatRef()) { return true; } } return false; } /*! * \brief Sets the start state of the FSM. * \param state The state to set as the start state. */ void SetStartState(int state) { XGRAMMAR_DCHECK(state < NumStates()); start_ = state; } /*! * \brief Adds an end/accepting state to the FSM. * \param state The state to add as an end state. */ void AddEndState(int state) { XGRAMMAR_DCHECK(state < NumStates()); ends_[state] = true; } /*! * \brief Adds a new state to the FSM and marks it as non-end. * \return The index of the newly added state. */ int AddState() { ends_.push_back(false); return fsm_.AddState(); } /*! * \brief Sets the end states of the FSM. * \param ends The new end states. */ void SetEndStates(const std::vector& ends) { ends_ = ends; } /*! \brief Returns the total number of states in the FSM. */ int NumStates() const { return fsm_.NumStates(); } /*! * \brief Access the methods of the underlying FSM. */ FSMType& GetFsm() { return fsm_; } /****************** FSM Traversal Algorithms ******************/ /*! * \brief Check if the FSM accepts the string. * \param str The input string. * \return True if the FSM accepts the string, false otherwise. */ bool AcceptString(const std::string& str) const; /*! * \brief Get the reachable states from the start state. * \param result The reachable states. The result is cleared at the beginning. */ void GetReachableStates(std::unordered_set* result) const; /*! * \brief Check if the FSM is a leaf FSM. * \return True if the FSM is a leaf FSM, false otherwise. */ bool IsLeaf() const; protected: /*! \brief The underlying finite state machine. */ FSMType fsm_; /*! \brief The start state of the FSM. */ int start_; /*! \brief The set of accepting/end states. */ std::vector ends_; protected: /*! \brief Whether this FSM is a deterministic finite automaton. */ bool is_dfa_ = false; }; /*! * \brief FSMWithStartEnd represents a FSM with start and end states. * \details It stores a pointer to a FSM, a start state, and a set of end states. Multiple * FSMWithStartEnd can share the same FSM. It also provides a set of methods to construct FSMs. */ class FSMWithStartEnd : public FSMWithStartEndBase { public: using FSMWithStartEndBase::FSMWithStartEndBase; /*! * \brief Convert the FSMWithStartEnd to a string. Only considers the nodes approachable from the * start state. * \return The string representation of the FSMWithStartEnd. */ std::string ToString() const; friend std::ostream& operator<<(std::ostream& os, const FSMWithStartEnd& fsm); /****************** FSM Construction Methods ******************/ /*! * \brief Return a copy of the FSMWithStartEnd. */ FSMWithStartEnd Copy() const; /*! * \brief Rebuild the FSM with the new state ids. * \param state_mapping The mapping from old state ids to new state ids. * \param new_num_states The new number of states. */ FSMWithStartEnd RebuildWithMapping(const std::vector& state_mapping, int new_num_states) const; /*! * \brief Add the underlying FSM to another complete FSM that could contain multiple FSMs. * Return a new FSMWithStartEnd that points to the complete FSM and whose start and ends are * mapped to the states in the complete FSM. * \param complete_fsm The complete FSM. * \param state_mapping The mapping from the old state ids to the new state ids. The result is * cleared at the beginning. Should not be nullptr. * \return The FSMWithStartEnd that points to the complete FSM. */ FSMWithStartEndWithSize AddToCompleteFSM(FSM* complete_fsm, std::vector* state_mapping); /*! * \brief Transform the FSMWithStartEnd to a CompactFSMWithStartEnd. * \return The CompactFSMWithStartEnd. */ CompactFSMWithStartEnd ToCompact(); /****************** FSM Algorithms ******************/ /*! * \brief Return a new FSM representing FSM* * \return The FSM that accepts FSM*. */ FSMWithStartEnd Star() const; /*! * \brief Return a new FSM representing rule1+. * \return The FSM that accepts rule1+. */ FSMWithStartEnd Plus() const; /*! * \brief Return a new FSM representing rule1?. * \return The FSM that accepts rule1?. */ FSMWithStartEnd Optional() const; /*! * \brief Return a new FSM representing the complement of the language. * \return The complement FSM. */ Result Not(int max_result_num_states = 1e6) const; /*! * \brief Intersect the FSMs. * \param lhs The left FSM. * \param rhs The right FSM. * \return The intersection of the FSMs. */ static Result Intersect( const FSMWithStartEnd& lhs, const FSMWithStartEnd& rhs, int max_result_num_states = 1e6 ); /*! * \brief Union the FSMs. * \param fsms The FSMs to be unioned. * \return The union of the FSMs. */ static FSMWithStartEnd Union(const std::vector& fsms); /*! * \brief Concatenate the FSMs. * \param fsms The FSMs to be concatenated, which should be in order. * \return The concatenation of the FSMs. */ static FSMWithStartEnd Concat(const std::vector& fsms); /*! * \brief Check if the FSM is a DFA. * \return True if the FSM is a DFA, false otherwise. */ bool IsDFA(); /*! * \brief Merge some states by removing some epsilon transitions. * \details If a --\epsilon--> b, and either 1) b doesn't have any other inward edges, or * 2) a doesn't have any other outward edges, we can merge a and b. */ FSMWithStartEnd SimplifyEpsilon(int max_num_states = 1e8) const; /*! * \brief Merge equivalent states in the FSM. * \details If two states are 1) pointed to by edges with the same label from the same state, and * 2) they are not pointed to by other edges, then we can merge them. * \example n0 --(c)--> n1, n0 --(c)--> n2, then we can merge n1 and n2. */ FSMWithStartEnd MergeEquivalentStates(int max_num_states = 1e5) const; /*! * \brief Transform the FSM to a DFA. * \param max_result_num_states The maximum number of states in the DFA. * \return The DFA. */ Result ToDFA(int max_num_states = 1e3) const; /*! * \brief Minimize the DFA. * \param max_result_num_states The maximum number of states in the DFA. * \return The minimized DFA. */ Result MinimizeDFA(int max_num_states = 1e3) const; }; /*! * \brief Wrapper that bundles an FSMWithStartEnd with explicit size metadata. It is * used when we want to store the number of edges and nodes in the part of the FSM, instead * of the completed FSMWithStartEnd. */ class FSMWithStartEndWithSize { public: // For serialization only FSMWithStartEndWithSize() = default; explicit FSMWithStartEndWithSize(FSMWithStartEnd fsm, int edge_num, int node_num) : fsm_(std::move(fsm)), edge_num_(edge_num), node_num_(node_num) {} const FSMWithStartEnd& GetFsm() const { return fsm_; } int GetEdgeNum() const { return edge_num_; } int GetNodeNum() const { return node_num_; } private: FSMWithStartEnd fsm_; int edge_num_ = 0; int node_num_ = 0; }; /*! * \brief A class that represents a compact-form FSM with a start state and a set of end states. * \details CompactFSMWithStartEnd stores a pointer to a CompactFSM, a start state, and a set of end * states. Multiple CompactFSMWithStartEnd can share the same CompactFSM. It share the same set of * visitor methods with FSMWithStartEnd. */ class CompactFSMWithStartEnd : public FSMWithStartEndBase { public: // For serialization only CompactFSMWithStartEnd() = default; explicit CompactFSMWithStartEnd(const CompactFSM& fsm, int start, const std::vector& ends) : FSMWithStartEndBase(fsm, start, ends), edge_num_(fsm.GetNumEdges()) {} using FSMWithStartEndBase::FSMWithStartEndBase; /*! * \brief Convert the FSMWithStartEnd to a string. Only considers the nodes approachable from the * start state. * \return The string representation of the FSMWithStartEnd. */ std::string ToString() const; /*! * \brief Transform the CompactFSMWithStartEnd to a FSMWithStartEnd. * \return The FSMWithStartEnd. */ FSMWithStartEnd ToFSM() const; private: size_t edge_num_ = 0; /*! * \brief Print the CompactFSMWithStartEnd. * \param os The output stream. * \param fsm The CompactFSMWithStartEnd. * \return The output stream. */ friend std::ostream& operator<<(std::ostream& os, const CompactFSMWithStartEnd& fsm); /*! * \brief Get the memory size of the CompactFSMWithStartEnd. * \param self The CompactFSMWithStartEnd. * \return The memory size of the CompactFSMWithStartEnd. */ friend std::size_t MemorySize(const CompactFSMWithStartEnd& self); friend struct member_trait; friend struct CompactFSMWithStartEndSerializeHelper; friend picojson::value SerializeJSONValue(const CompactFSMWithStartEnd& value); friend std::optional DeserializeJSONValue( CompactFSMWithStartEnd* result, const picojson::value& value, const std::string& type_name ); }; /*! * \brief Wrapper that bundles a CompactFSMWithStartEnd with explicit size metadata. It is * used when we want to store the number of edges and nodes in the part of the * CompactFSMWithStartEnd, instead of the completed CompactFSMWithStartEnd. */ class CompactFSMWithStartEndWithSize { public: // For serialization only CompactFSMWithStartEndWithSize() = default; explicit CompactFSMWithStartEndWithSize(CompactFSMWithStartEnd fsm, int edge_num, int node_num) : fsm_(std::move(fsm)), edge_num_(edge_num), node_num_(node_num) {} const CompactFSMWithStartEnd& GetFsm() const { return fsm_; } int GetEdgeNum() const { return edge_num_; } int GetNodeNum() const { return node_num_; } friend picojson::value SerializeJSONValue(const CompactFSMWithStartEndWithSize& value); friend std::optional DeserializeJSONValue( CompactFSMWithStartEndWithSize* result, const picojson::value& value, const std::string& type_name ); private: CompactFSMWithStartEnd fsm_; int edge_num_ = 0; int node_num_ = 0; friend std::size_t MemorySize(const CompactFSMWithStartEndWithSize& self) { return MemorySize(self.fsm_) + sizeof(self.edge_num_) + sizeof(self.node_num_); } friend struct CompactFSMWithStartEndWithSizeSerializeHelper; }; std::optional DeserializeJSONValue( CompactFSMWithStartEndWithSize* result, const picojson::value& value, const std::string& type_name = "" ); /****************** FSMWithStartEndBase Template Implementation ******************/ template inline bool FSMWithStartEndBase::AcceptString(const std::string& str) const { std::unordered_set start_states{start_}; fsm_.GetEpsilonClosure(&start_states); std::unordered_set result_states; for (const auto& character : str) { result_states.clear(); fsm_.Advance( start_states, static_cast(static_cast(character)), &result_states, FSMEdge::EdgeType::kCharRange, false ); if (result_states.empty()) { return false; } start_states = result_states; } return std::any_of(start_states.begin(), start_states.end(), [&](int state) { return ends_[state]; }); } template inline void FSMWithStartEndBase::GetReachableStates(std::unordered_set* result ) const { return fsm_.GetReachableStates({start_}, result); } template inline bool FSMWithStartEndBase::IsLeaf() const { std::unordered_set reachable_states; GetReachableStates(&reachable_states); for (const auto& state : reachable_states) { for (const auto& edge : fsm_.GetEdges(state)) { if (edge.IsRuleRef() || edge.IsRepeatRef()) { return false; } } } return true; } } // namespace xgrammar #endif // XGRAMMAR_FSM_H_ xgrammar-0.2.3/cpp/fsm_builder.cc000066400000000000000000000720051521764210300167230ustar00rootroot00000000000000/*! * Copyright (c) 2025 by Contributors * \file xgrammar/fsm_builder.cc */ #include "fsm_builder.h" #include #include #include #include #include #include #include #include #include #include #include #include "fsm.h" #include "support/logging.h" #include "support/utils.h" namespace xgrammar { class RegexIR { public: struct Leaf; struct Symbol; struct Union; struct Bracket; struct Repeat; static constexpr int kRepeatNoUpperBound = -1; using State = std::variant; // This struct is used to store the string in regex, or // the character class in regex. struct Leaf { std::string regex; }; // This struct is used to store the symbol in regex, i.e. // +, *, ? enum class RegexSymbol { star, plus, optional, }; struct Bracket { std::vector states; }; struct Symbol { RegexSymbol symbol; std::vector state; }; // This struct is used to represent a union symbol. struct Union { std::vector states; }; struct Repeat { std::vector states; int lower_bound = 0; int upper_bound = 0; }; struct LookAhead { bool is_positive; std::vector states; }; // This struct is used to represent a bracket in regex. std::vector states; /*! \brief Constructs a NFA from the regex IR. */ Result Build() const; /*! \brief the visit function for the variant. */ Result visit(const Leaf& state) const; Result visit(const Symbol& state) const; Result visit(const Union& state) const; Result visit(const Bracket& state) const; Result visit(const Repeat& state) const; Result visit(const LookAhead& state) const; private: /*! * \brief Construct a FSM from a regex string. * \details The regex string should only be the format like "abx" or [a-c0-9]. * \details Any symbols like "a|b" or "a*b" are not supported. * \param regex The regex string. * \return The FSM with start and end states. */ static FSMWithStartEnd BuildLeafFSMFromRegex(const std::string& regex); /*! * \brief Handle escape characters. * \param regex the corresponding string. * \param start the pos escape characters start. */ static std::vector> HandleEscapes(const std::string& regex, int start); /*! * \brief Check repeat in regex. i.e {...} and {...,...} * \param regex The regex string. * \param start The start position of the repeat. i.e. regex[start] == '{'. * After the function, start will be the position of '}'. * \return The repeat range. */ static Result> CheckRepeat(const std::string& regex, int& start); friend class RegexFSMBuilder; }; Result> RegexIR::CheckRepeat(const std::string& regex, int& start) { if (regex[start] != '{') { return ResultErr("Invalid repeat format1"); } int lower_bound = 0; int upper_bound = RegexIR::kRepeatNoUpperBound; std::string num_str; XGRAMMAR_DCHECK(regex[start] == '{'); start++; while (static_cast(start) < regex.size() && regex[start] == ' ') { start++; } while (static_cast(start) < regex.size() && std::isdigit(regex[start])) { num_str += regex[start]; start++; } if (num_str.empty()) { return ResultErr("Invalid repeat format2"); } lower_bound = std::stoi(num_str); while (static_cast(start) < regex.size() && regex[start] == ' ') { start++; } // The format is {n} if (regex[start] == '}') { upper_bound = lower_bound; return ResultOk(std::make_pair(lower_bound, upper_bound)); } if (regex[start] != ',') { return ResultErr("Invalid repeat format3"); } XGRAMMAR_DCHECK(regex[start] == ','); start++; while (static_cast(start) < regex.size() && regex[start] == ' ') { start++; } // The format is {n,} if (regex[start] == '}') { return ResultOk(std::make_pair(lower_bound, upper_bound)); } num_str.clear(); while (static_cast(start) < regex.size() && std::isdigit(regex[start])) { num_str += regex[start]; start++; } if (num_str.empty()) { return ResultErr("Invalid repeat format4"); } upper_bound = std::stoi(num_str); while (static_cast(start) < regex.size() && regex[start] == ' ') { start++; } if (regex[start] != '}') { return ResultErr("Invalid repeat format5"); } XGRAMMAR_DCHECK(regex[start] == '}'); return ResultOk(std::make_pair(lower_bound, upper_bound)); } Result RegexIR::Build() const { if (states.empty()) { FSM empty_fsm(1); FSMWithStartEnd result(empty_fsm, 0, {true}, false); return ResultOk(std::move(result)); } std::vector fsm_list; for (const auto& state : states) { auto visited = std::visit([&](auto&& arg) { return visit(arg); }, state); if (visited.IsErr()) { return visited; } fsm_list.push_back(std::move(visited).Unwrap()); } if (fsm_list.size() > 1) { return ResultOk(FSMWithStartEnd::Concat(fsm_list)); } else { // If there is only one FSM, return it directly. return ResultOk(std::move(fsm_list[0])); } } Result RegexIR::visit(const RegexIR::Leaf& state) const { FSMWithStartEnd result = BuildLeafFSMFromRegex(state.regex); return ResultOk(std::move(result)); } Result RegexIR::visit(const RegexIR::Union& state) const { std::vector fsm_list; for (const auto& child : state.states) { auto visited = std::visit([&](auto&& arg) { return RegexIR::visit(arg); }, child); if (visited.IsErr()) { return visited; } fsm_list.push_back(std::move(visited).Unwrap()); } if (fsm_list.size() <= 1) { return ResultErr("Invalid union"); } return ResultOk(FSMWithStartEnd::Union(fsm_list)); } Result RegexIR::visit(const RegexIR::Symbol& state) const { if (state.state.size() != 1) { return ResultErr("Invalid symbol"); } Result child_result = std::visit([&](auto&& arg) { return RegexIR::visit(arg); }, state.state[0]); if (child_result.IsErr()) { return child_result; } auto child = std::move(child_result).Unwrap(); switch (state.symbol) { case RegexIR::RegexSymbol::plus: { return ResultOk(child.Plus()); } case RegexIR::RegexSymbol::star: { return ResultOk(child.Star()); } case RegexIR::RegexSymbol::optional: { return ResultOk(child.Optional()); } default: { XGRAMMAR_LOG(FATAL) << "Unknown regex symbol: " << static_cast(state.symbol); } } } Result RegexIR::visit(const RegexIR::Bracket& state) const { std::vector fsm_list; for (const auto& child : state.states) { auto visited = std::visit([&](auto&& arg) { return RegexIR::visit(arg); }, child); if (visited.IsErr()) { return visited; } fsm_list.push_back(std::move(visited).Unwrap()); } if (fsm_list.empty()) { return ResultErr("Invalid bracket"); } return ResultOk(FSMWithStartEnd::Concat(fsm_list)); } Result RegexIR::visit(const RegexIR::Repeat& state) const { if (state.states.size() != 1) { return ResultErr("Invalid repeat"); } Result child_result = std::visit([&](auto&& arg) { return RegexIR::visit(arg); }, state.states[0]); if (child_result.IsErr()) { return child_result; } FSMWithStartEnd child = std::move(child_result).Unwrap(); FSMWithStartEnd result = child.Copy(); std::unordered_set new_ends; if (state.lower_bound == 1) { // Insert the first end state. for (int end = 0; end < result.NumStates(); ++end) { if (result.IsEndState(end)) { new_ends.insert(end); } } } // Handling {n,} if (state.upper_bound == RegexIR::kRepeatNoUpperBound) { for (int i = 2; i < state.lower_bound; i++) { result = FSMWithStartEnd::Concat(std::vector{result, child}); } int end_state_of_lower_bound_fsm = -1; for (int end = 0; end < result.NumStates(); ++end) { if (result.IsEndState(end)) { end_state_of_lower_bound_fsm = end; break; } } XGRAMMAR_DCHECK(end_state_of_lower_bound_fsm != -1) << "No end state found in the lower bound FSM."; result = FSMWithStartEnd::Concat(std::vector{result, child}); for (int end = 0; end < result.NumStates(); ++end) { if (result.IsEndState(end)) { result.GetFsm().AddEpsilonEdge(end, end_state_of_lower_bound_fsm); } } return ResultOk(std::move(result)); } // Handling {n, m} or {n} for (int i = 2; i <= state.upper_bound; i++) { result = FSMWithStartEnd::Concat(std::vector{result, child}); if (i >= state.lower_bound) { for (int end = 0; end < result.NumStates(); ++end) { if (result.IsEndState(end)) { new_ends.insert(end); } } } } for (const auto& end : new_ends) { result.AddEndState(end); } return ResultOk(std::move(result)); } FSMWithStartEnd RegexIR::BuildLeafFSMFromRegex(const std::string& regex) { FSM empty_fsm(0); FSMWithStartEnd result(empty_fsm, 0, {}, true); // Handle the regex string. if (!(regex[0] == '[' && regex[regex.size() - 1] == ']')) { result.AddState(); for (size_t i = 0; i < regex.size(); i++) { if (regex[i] != '\\') { if (regex[i] == '.') { result.GetFsm().AddEdge(result.NumStates() - 1, result.NumStates(), 0, 0xFF); } else { result.GetFsm().AddEdge( result.NumStates() - 1, result.NumStates(), static_cast(regex[i]), static_cast(regex[i]) ); } result.AddState(); continue; } std::vector> escape_vector = HandleEscapes(regex, i); for (const auto& escape : escape_vector) { result.GetFsm().AddEdge( result.NumStates() - 1, result.NumStates(), static_cast(escape.first), static_cast(escape.second) ); } result.AddState(); i++; } result.AddEndState(result.NumStates() - 1); } else if (regex[0] == '[' && regex[regex.size() - 1] == ']') { // Handle the character class. result.AddState(); result.AddState(); result.AddEndState(1); bool reverse = regex[1] == '^'; for (size_t i = reverse ? 2 : 1; i < regex.size() - 1; i++) { if (regex[i] != '\\') { if (!(((i + 2) < regex.size() - 1) && regex[i + 1] == '-')) { // A single char. result.GetFsm().AddEdge( 0, 1, static_cast(regex[i]), static_cast(regex[i]) ); continue; } // Handle the char range. if (regex[i + 2] != '\\') { result.GetFsm().AddEdge( 0, 1, static_cast(regex[i]), static_cast(regex[i + 2]) ); i = i + 2; continue; } auto escaped_edges = HandleEscapes(regex, i + 2); // Means it's not a range. if (escaped_edges.size() != 1 || escaped_edges[0].first != escaped_edges[0].second) { result.GetFsm().AddEdge( 0, 1, static_cast(regex[i]), static_cast(regex[i]) ); continue; } result.GetFsm().AddEdge( 0, 1, static_cast(regex[0]), static_cast(escaped_edges[0].first) ); i = i + 3; continue; } auto escaped_edges = HandleEscapes(regex, i); i = i + 1; if (escaped_edges.size() != 1 || escaped_edges[0].first != escaped_edges[0].second) { // It's a multi-match escape char. for (const auto& edge : escaped_edges) { result.GetFsm().AddEdge( 0, 1, static_cast(edge.first), static_cast(edge.second) ); } continue; } if (!(((i + 2) < regex.size() - 1) && regex[i + 1] == '-')) { result.GetFsm().AddEdge( 0, 1, static_cast(escaped_edges[0].first), static_cast(escaped_edges[0].second) ); continue; } if (regex[i + 2] != '\\') { result.GetFsm().AddEdge( 0, 1, static_cast(escaped_edges[0].first), static_cast(regex[i + 2]) ); i = i + 2; continue; } auto rhs_escaped_edges = HandleEscapes(regex, i + 2); if (rhs_escaped_edges.size() != 1 || rhs_escaped_edges[0].first != rhs_escaped_edges[0].second) { result.GetFsm().AddEdge( 0, 1, static_cast(escaped_edges[0].first), static_cast(escaped_edges[0].second) ); continue; } result.GetFsm().AddEdge( 0, 1, static_cast(escaped_edges[0].first), static_cast(rhs_escaped_edges[0].first) ); i = i + 3; continue; } bool has_edge[0x100]; memset(has_edge, 0, sizeof(has_edge)); FSM new_fsm(2); for (const auto& edge : result.GetFsm().GetEdges(0)) { for (int i = edge.min; i <= edge.max; i++) { has_edge[i] = true; } } // Simplify the edges. e.g [abc] -> [a-c] int last = -1; if (reverse) { for (int i = 0; i < 0x100; i++) { if (!has_edge[i]) { if (last == -1) { last = i; } continue; } if (last != -1) { new_fsm.AddEdge(0, 1, last, i - 1); last = -1; } } if (last != -1) { new_fsm.AddEdge(0, 1, last, 0xFF); } } else { for (int i = 0; i < 0x100; i++) { if (has_edge[i]) { if (last == -1) { last = i; } continue; } if (last != -1) { new_fsm.AddEdge(0, 1, last, i - 1); last = -1; } } if (last != -1) { new_fsm.AddEdge(0, 1, last, 0xFF); } } std::vector ends(new_fsm.NumStates(), false); ends[1] = true; result = FSMWithStartEnd(new_fsm, 0, ends, false); } else { // TODO: The support for rules. XGRAMMAR_LOG(WARNING) << "rule is not supported yet."; } return result; } std::vector> RegexIR::HandleEscapes(const std::string& regex, int start) { std::vector> result; switch (regex[start + 1]) { case 'n': { return std::vector>(1, std::make_pair('\n', '\n')); } case 't': { return std::vector>(1, std::make_pair('\t', '\t')); } case 'r': { return std::vector>(1, std::make_pair('\r', '\r')); } case '0': { return std::vector>(1, std::make_pair('\0', '\0')); } case 's': { return std::vector>(1, std::make_pair(0, ' ')); } case 'S': { return std::vector>(1, std::make_pair(' ' + 1, 0x00FF)); } case 'd': { return std::vector>(1, std::make_pair('0', '9')); } case 'D': { std::vector> result; result.emplace_back(0, '0' - 1); result.emplace_back('9' + 1, 0x00FF); return result; } case 'w': { std::vector> result; result.emplace_back('0', '9'); result.emplace_back('a', 'z'); result.emplace_back('A', 'Z'); result.emplace_back('_', '_'); return result; } case 'W': { std::vector> result; result.emplace_back(0, '0' - 1); result.emplace_back('9' + 1, 'A' - 1); result.emplace_back('Z' + 1, '_' - 1); result.emplace_back('_' + 1, 'a' - 1); result.emplace_back('z' + 1, 0x00FF); return result; } default: { return std::vector>( 1, std::make_pair(regex[start + 1], regex[start + 1]) ); } } } Result RegexFSMBuilder::Build(const std::string& regex) { RegexIR ir; using IRState = std::variant; // We use a stack to store the states. std::stack stack; int left_middle_bracket = -1; for (int i = 0; i < static_cast(regex.size()); i++) { if (i == 0 && regex[i] == '^') { continue; } if (i == static_cast(regex.size()) - 1 && regex[i] == '$') { continue; } // Handle The class. if (regex[i] == '[') { if (left_middle_bracket != -1) { return ResultErr("Nested middle bracket!"); } left_middle_bracket = i; continue; } if (regex[i] == ']') { if (left_middle_bracket == -1) { return ResultErr("Invalid middle bracket!"); } RegexIR::Leaf leaf; leaf.regex = regex.substr(left_middle_bracket, i - left_middle_bracket + 1); stack.push(leaf); left_middle_bracket = -1; continue; } if (left_middle_bracket != -1) { if (regex[i] == '\\') { i++; } continue; } if (regex[i] == '+' || regex[i] == '*' || regex[i] == '?') { if (stack.empty()) { return ResultErr("Invalid regex: no state before operator!"); } auto state = stack.top(); if (std::holds_alternative(state)) { return ResultErr("Invalid regex: no state before operator!"); } stack.pop(); auto child = std::get(state); RegexIR::Symbol symbol; symbol.state.push_back(child); switch (regex[i]) { case '+': { symbol.symbol = RegexIR::RegexSymbol::plus; break; } case '*': { symbol.symbol = RegexIR::RegexSymbol::star; break; } case '?': { symbol.symbol = RegexIR::RegexSymbol::optional; break; } } stack.push(symbol); continue; } if (regex[i] == '(' || regex[i] == '|') { stack.push(regex[i]); if (i < static_cast(regex.size()) - 2 && regex[i] == '(' && regex[i + 1] == '?' && regex[i + 2] == ':') { i += 2; continue; } if (i < static_cast(regex.size()) - 2 && regex[i] == '(' && regex[i + 1] == '?' && (regex[i + 2] == '!' || regex[i + 2] == '=')) { i += 2; // TODO(Linzhang Li): Handling the lookahead. continue; } continue; } if (regex[i] == ')') { std::stack states; bool paired = false; bool unioned = false; while ((!stack.empty()) && (!paired)) { auto state = stack.top(); stack.pop(); if (std::holds_alternative(state)) { char c = std::get(state); if (c == '(') { paired = true; break; } if (c == '|') { unioned = true; } states.push(state); } else { states.push(state); } } if (!paired) { return ResultErr("Invalid regex: no paired bracket!" + std::to_string(__LINE__)); } if (states.empty()) { continue; } if (!unioned) { RegexIR::Bracket bracket; while (!states.empty()) { auto state = states.top(); states.pop(); auto child = std::get(state); bracket.states.push_back(child); } stack.push(bracket); } else { RegexIR::Union union_state; RegexIR::Bracket bracket; while (!states.empty()) { auto state = states.top(); states.pop(); if (std::holds_alternative(state)) { char c = std::get(state); if (c == '|') { union_state.states.push_back(bracket); bracket.states.clear(); continue; } return ResultErr("Invalid regex: no paired bracket!" + std::to_string(__LINE__)); } if (std::holds_alternative(state)) { auto child = std::get(state); bracket.states.push_back(child); continue; } return ResultErr("Invalid regex: no paired bracket!" + std::to_string(__LINE__)); } union_state.states.push_back(bracket); stack.push(union_state); } continue; } if (regex[i] == '{') { if (stack.empty()) { return ResultErr("Invalid regex: no state before repeat!"); } auto state = stack.top(); if (std::holds_alternative(state)) { return ResultErr("Invalid regex: no state before repeat!"); } stack.pop(); auto bounds_result = RegexIR::CheckRepeat(regex, i); if (bounds_result.IsErr()) { return ResultErr(std::move(bounds_result).UnwrapErr()); } auto bounds = std::move(bounds_result).Unwrap(); auto child = std::get(state); RegexIR::Repeat repeat; repeat.lower_bound = bounds.first; repeat.upper_bound = bounds.second; repeat.states.push_back(child); stack.push(repeat); continue; } RegexIR::Leaf leaf; if (regex[i] != '\\') { leaf.regex = regex[i]; } else { leaf.regex = regex.substr(i, 2); i++; } stack.push(leaf); continue; } std::vector res_states; std::vector union_state_list; bool unioned = false; while (!stack.empty()) { if (std::holds_alternative(stack.top())) { char c = std::get(stack.top()); if (c == '|') { union_state_list.push_back(res_states); res_states.clear(); unioned = true; stack.pop(); continue; } return ResultErr("Invalid regex: no paired!"); } auto state = stack.top(); stack.pop(); auto child = std::get(state); res_states.push_back(std::move(child)); } if (!unioned) { for (auto it = res_states.rbegin(); it != res_states.rend(); ++it) { ir.states.push_back(std::move(*it)); } } else { union_state_list.push_back(res_states); RegexIR::Union union_state; for (auto it = union_state_list.begin(); it != union_state_list.end(); ++it) { RegexIR::Bracket bracket; for (auto state = it->rbegin(); state != it->rend(); ++state) { bracket.states.push_back(std::move(*state)); } union_state.states.push_back(std::move(bracket)); } ir.states.push_back(std::move(union_state)); } return ir.Build(); } class TrieFSMBuilderImpl { public: TrieFSMBuilderImpl() = default; std::optional Build( const std::vector& patterns, const std::vector& excluded_patterns, std::vector* end_states, bool allow_overlap, bool add_back_edges ); void AddBackEdges(FSM* fsm, int start, const std::unordered_set& ends); }; std::optional TrieFSMBuilderImpl::Build( const std::vector& patterns, const std::vector& excluded_patterns, std::vector* end_states, bool allow_overlap, bool add_back_edges ) { FSM fsm(1); int start = 0; std::unordered_set ends; if (end_states) { end_states->clear(); } for (const auto& pattern : patterns) { // Check for empty patterns if (!allow_overlap && pattern.empty()) { return std::nullopt; } int current_state = 0; for (const auto& ch : pattern) { int32_t ch_int32 = static_cast(static_cast(ch)); int next_state = fsm.GetNextState(current_state, ch_int32); if (next_state == FSM::kNoNextState) { next_state = fsm.AddState(); fsm.AddEdge(current_state, next_state, ch_int32, ch_int32); } current_state = next_state; if (!allow_overlap && ends.count(current_state) > 0) { return std::nullopt; } } if (!allow_overlap && fsm.GetEdges(current_state).size() > 0) { return std::nullopt; } ends.insert(current_state); if (end_states) { end_states->push_back(current_state); } } std::unordered_set dead_state_set; if (add_back_edges) { // Build trie for excluded patterns. for (const auto& excluded_pattern : excluded_patterns) { if (!allow_overlap && excluded_pattern.empty()) { return std::nullopt; } int current_state = 0; for (const auto& ch : excluded_pattern) { int32_t ch_int32 = static_cast(static_cast(ch)); int next_state = fsm.GetNextState(current_state, ch_int32); if (next_state == FSM::kNoNextState) { next_state = fsm.AddState(); fsm.AddEdge(current_state, next_state, ch_int32, ch_int32); } current_state = next_state; if (!allow_overlap && ends.count(current_state) > 0) { return std::nullopt; } } if (!allow_overlap && fsm.GetEdges(current_state).size() > 0) { return std::nullopt; } ends.insert(current_state); dead_state_set.insert(current_state); } // Add back edges. AddBackEdges(&fsm, start, ends); // Remove the edges to excluded end states. if (dead_state_set.size() != 0) { for (int state = 0; state < fsm.NumStates(); state++) { std::vector& edges = fsm.GetEdges(state); std::vector new_edges; for (const auto& edge : edges) { if (dead_state_set.count(edge.target) == 0) { new_edges.push_back(edge); } } edges = std::move(new_edges); } } } else if (excluded_patterns.size() > 0) { XGRAMMAR_LOG(WARNING) << "Excluded patterns are ignored when back edges are not added."; } std::vector is_end_state(fsm.NumStates(), false); for (const auto& end : ends) { is_end_state[end] = true; } return FSMWithStartEnd(fsm, start, is_end_state); } void TrieFSMBuilderImpl::AddBackEdges(FSM* fsm, int start, const std::unordered_set& ends) { // Build an Aho-Corasick automaton by adding back edges. // When matching on the trie fails, we should go back to the start state and // find the next match. Back edges represent such state transitions. auto f_add_range_edges = [&](int node, std::set& cur_edges_set) { cur_edges_set.insert(FSMEdge(-1, -1, 0)); cur_edges_set.insert(FSMEdge(256, 256, 0)); XGRAMMAR_DCHECK(cur_edges_set.size() >= 2); for (auto it = std::next(cur_edges_set.begin()); it != cur_edges_set.end(); ++it) { FSMEdge prev_edge = *std::prev(it); XGRAMMAR_DCHECK(prev_edge.max < it->min); if (prev_edge.max + 1 != it->min) { auto new_edge = FSMEdge(prev_edge.max + 1, it->min - 1, start); // The new edge should be inserted before the current edge to avoid infinite loop. XGRAMMAR_DCHECK(new_edge < *it); cur_edges_set.insert(new_edge); } } // Remove first and last element of cur_edges_set XGRAMMAR_DCHECK(*cur_edges_set.begin() == FSMEdge(-1, -1, 0)); XGRAMMAR_DCHECK(*std::prev(cur_edges_set.end()) == FSMEdge(256, 256, 0)); cur_edges_set.erase(cur_edges_set.begin()); cur_edges_set.erase(std::prev(cur_edges_set.end())); XGRAMMAR_DCHECK(cur_edges_set.begin()->min == 0); XGRAMMAR_DCHECK(std::prev(cur_edges_set.end())->max == 255); }; for (int i = 0; i < fsm->NumStates(); i++) { if (i == start || ends.count(i) > 0) { continue; } std::vector& cur_edges = fsm->GetEdges(i); XGRAMMAR_DCHECK(cur_edges.size() > 0); std::set cur_edges_set(cur_edges.begin(), cur_edges.end()); // Step 1. Add edges in the edges of the start state. // For start--(c)-->t, add i--(c)-->t. const auto& root_edges = fsm->GetEdges(start); for (const auto& root_edge : root_edges) { XGRAMMAR_DCHECK(root_edge.min == root_edge.max); if (cur_edges_set.count(root_edge) == 0) { cur_edges_set.insert(root_edge); } } // Step 2. Add i--(c)-->start for c not in the edge set of i. f_add_range_edges(i, cur_edges_set); // Step 3. Update the edges of i. cur_edges.clear(); cur_edges.insert(cur_edges.end(), cur_edges_set.begin(), cur_edges_set.end()); } // Finally, add range edges to the start state. std::vector& start_edges = fsm->GetEdges(start); std::set start_edges_set(start_edges.begin(), start_edges.end()); f_add_range_edges(start, start_edges_set); start_edges.clear(); start_edges.insert(start_edges.end(), start_edges_set.begin(), start_edges_set.end()); } std::optional TrieFSMBuilder::Build( const std::vector& patterns, const std::vector& exclude_patterns, std::vector* end_states, bool allow_overlap, bool add_back_edges ) { return TrieFSMBuilderImpl().Build( patterns, exclude_patterns, end_states, allow_overlap, add_back_edges ); } } // namespace xgrammar xgrammar-0.2.3/cpp/fsm_builder.h000066400000000000000000000033571521764210300165710ustar00rootroot00000000000000/*! * Copyright (c) 2025 by Contributors * \file xgrammar/fsm_builder.h */ #ifndef XGRAMMAR_FSM_BUILDER_H_ #define XGRAMMAR_FSM_BUILDER_H_ #include #include #include #include "fsm.h" #include "support/utils.h" namespace xgrammar { /*! * \brief A builder that converts a regex string to a FSM. */ class RegexFSMBuilder { public: /*! * \brief Converts a regex string to a FSM. * \param regex The regex string. * \return The FSM with start and end states. */ static Result Build(const std::string& regex); }; /*! * \brief A builder that converts a list of patterns to a trie-based FSM. */ class TrieFSMBuilder { public: /*! * \brief Build a trie-based FSM from a list of patterns. * \param patterns The patterns to be built. * \param excluded_patterns The patterns to be excluded. * \param end_states The end states of the FSM. This is the terminal state of each pattern and * the order follows the order of patterns. * \param allow_overlap Whether to allow overlap between patterns (one being a prefix of the * other). It does not allow empty patterns either. If false and there is overlap, will return * std::nullopt. * \param add_back_edges Whether to add back edges to the FSM. This complements the trie to an * Aho-Corasick automaton. * \return If success, the FSM with start and end states. Otherwise, std::nullopt. */ static std::optional Build( const std::vector& patterns, const std::vector& excluded_patterns, std::vector* end_states = nullptr, bool allow_overlap = true, bool add_back_edges = false ); }; } // namespace xgrammar #endif // XGRAMMAR_FSM_BUILDER_H_ xgrammar-0.2.3/cpp/grammar.cc000066400000000000000000000140121521764210300160500ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar.cc */ #include #include #include "grammar_functor.h" #include "grammar_parser.h" #include "grammar_printer.h" #include "json_schema_converter.h" #include "regex_converter.h" #include "structural_tag.h" #include "support/json_serializer.h" #include "support/logging.h" #include "xgrammar/exception.h" namespace xgrammar { /******************* Grammar::Impl *******************/ std::size_t MemorySize(const Grammar::Impl& impl) { /// TODO: Now, we evaluatve memory size of rule strings as sizeof(std::string), /// with an assumption that the string is small. /// This should be improved in the future. return impl.rules_.size() * sizeof(std::string) + MemorySize(impl.grammar_expr_data_) + MemorySize(impl.grammar_expr_indptr_) + MemorySize(impl.complete_fsm) + MemorySize(impl.per_rule_fsms) + MemorySize(impl.allow_empty_rule_ids); } /******************* Grammar *******************/ std::string Grammar::ToString() const { return GrammarPrinter(*this).ToString(); } Grammar Grammar::FromEBNF(const std::string& ebnf_string, const std::string& root_rule_name) { auto grammar = ParseEBNF(ebnf_string, root_rule_name); grammar = GrammarNormalizer().Apply(grammar); return grammar; } Grammar Grammar::FromJSONSchema( const std::string& schema, bool any_whitespace, std::optional indent, std::optional> separators, bool strict_mode, std::optional max_whitespace_cnt, bool print_converted_ebnf, bool any_order ) { auto ebnf_string = JSONSchemaToEBNF( schema, any_whitespace, indent, separators, strict_mode, max_whitespace_cnt, JSONFormat::kJSON, any_order ); if (print_converted_ebnf) { XGRAMMAR_LOG(INFO) << "Converted EBNF: " << ebnf_string << std::endl; } return FromEBNF(ebnf_string); } Grammar Grammar::FromRegex(const std::string& regex, bool print_converted_ebnf) { auto ebnf_string = RegexToEBNF(regex); if (print_converted_ebnf) { XGRAMMAR_LOG(INFO) << "Converted EBNF: " << ebnf_string << std::endl; } return FromEBNF(ebnf_string); } std::variant Grammar::FromStructuralTag( const std::string& structural_tag_json, const std::optional& tokenizer_info ) { return StructuralTagToGrammar(structural_tag_json, tokenizer_info).ToVariant(); } // Optimized json grammar for the speed of the grammar matcher const std::string kJSONGrammarString = R"( root ::= ( "{" [ \n\t]* members_and_embrace | "[" [ \n\t]* elements_or_embrace ) value_non_str ::= ( "{" [ \n\t]* members_and_embrace | "[" [ \n\t]* elements_or_embrace | "0" fraction exponent | [1-9] [0-9]* fraction exponent | "-" [0-9] fraction exponent | "-" [1-9] [0-9]* fraction exponent | "true" | "false" | "null" ) (= [ \n\t]* member_suffix_suffix) members_and_embrace ::= ("\"" characters_and_colon [ \n\t]* members_suffix | "}") (= [ \n\t,}\]]) members_suffix ::= ( value_non_str [ \n\t]* member_suffix_suffix | "\"" characters_and_embrace | "\"" characters_and_comma [ \n\t]* "\"" characters_and_colon [ \n\t]* members_suffix ) (= [ \n\t,}\]]) member_suffix_suffix ::= ( "}" | "," [ \n\t]* "\"" characters_and_colon [ \n\t]* members_suffix ) (= [ \n\t,}\]]) elements_or_embrace ::= ( "{" [ \n\t]* members_and_embrace elements_rest [ \n\t]* "]" | "[" [ \n\t]* elements_or_embrace elements_rest [ \n\t]* "]" | "\"" characters_item elements_rest [ \n\t]* "]" | "0" fraction exponent elements_rest [ \n\t]* "]" | [1-9] [0-9]* fraction exponent elements_rest [ \n\t]* "]" | "-" "0" fraction exponent elements_rest [ \n\t]* "]" | "-" [1-9] [0-9]* fraction exponent elements_rest [ \n\t]* "]" | "true" elements_rest [ \n\t]* "]" | "false" elements_rest [ \n\t]* "]" | "null" elements_rest [ \n\t]* "]" | "]" ) elements ::= ( "{" [ \n\t]* members_and_embrace elements_rest | "[" [ \n\t]* elements_or_embrace elements_rest | "\"" characters_item elements_rest | "0" fraction exponent elements_rest | [1-9] [0-9]* fraction exponent elements_rest | "-" [0-9] fraction exponent elements_rest | "-" [1-9] [0-9]* fraction exponent elements_rest | "true" elements_rest | "false" elements_rest | "null" elements_rest ) elements_rest ::= ( "" | [ \n\t]* "," [ \n\t]* elements ) characters_and_colon ::= ( "\"" [ \n\t]* ":" | [^"\\\x00-\x1F] characters_and_colon | "\\" escape characters_and_colon ) (=[ \n\t]* [\"{[0-9tfn-]) characters_and_comma ::= ( "\"" [ \n\t]* "," | [^"\\\x00-\x1F] characters_and_comma | "\\" escape characters_and_comma ) (=[ \n\t]* "\"") characters_and_embrace ::= ( "\"" [ \n\t]* "}" | [^"\\\x00-\x1F] characters_and_embrace | "\\" escape characters_and_embrace ) (=[ \n\t]* [},]) characters_item ::= ( "\"" | [^"\\\x00-\x1F] characters_item | "\\" escape characters_item ) (= [ \n\t]* [,\]]) escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] fraction ::= "" | "." [0-9] [0-9]* exponent ::= "" | "e" sign [0-9] [0-9]* | "E" sign [0-9] [0-9]* sign ::= "" | "+" | "-" )"; Grammar Grammar::BuiltinJSONGrammar() { static const Grammar grammar = FromEBNF(kJSONGrammarString); return grammar; } Grammar Grammar::Union(const std::vector& grammars) { return GrammarUnionFunctor::Apply(grammars); } Grammar Grammar::Concat(const std::vector& grammars) { return GrammarConcatFunctor::Apply(grammars); } std::ostream& operator<<(std::ostream& os, const Grammar& grammar) { os << grammar.ToString(); return os; } std::string Grammar::SerializeJSON() const { return AutoSerializeJSON(*this, true); } std::variant Grammar::DeserializeJSON(const std::string& json_string) { Grammar result{NullObj()}; if (auto err = AutoDeserializeJSON(&result, json_string, true, "Grammar")) { return err.value(); } return result; } } // namespace xgrammar xgrammar-0.2.3/cpp/grammar_builder.cc000066400000000000000000000231231521764210300175610ustar00rootroot00000000000000/*! * Copyright (c) 2026 by Contributors * \file xgrammar/grammar_builder.cc */ #include "grammar_builder.h" #include #include #include #include "support/logging.h" namespace xgrammar { /****************** GrammarBuilder ******************/ GrammarBuilder::GrammarBuilder() : grammar_(std::make_shared()) {} GrammarBuilder::GrammarBuilder(const Grammar& grammar) : grammar_(std::make_shared(*grammar.operator->())) { for (int i = 0; i < static_cast(grammar->NumRules()); ++i) { auto rule = grammar->GetRule(i); rule_name_to_id_[rule.name] = i; } } Grammar GrammarBuilder::Get(const std::string& root_rule_name) { int32_t root_rule_id = GetRuleId(root_rule_name); XGRAMMAR_CHECK(root_rule_id != -1) << "The root rule with name \"" << root_rule_name << "\" is not found."; return Get(root_rule_id); } Grammar GrammarBuilder::Get(int32_t root_rule_id) { XGRAMMAR_CHECK(root_rule_id >= 0 && root_rule_id < static_cast(grammar_->rules_.size())) << "The root rule id " << root_rule_id << " is out of bound."; grammar_->root_rule_id_ = root_rule_id; return Grammar(grammar_); } int32_t GrammarBuilder::AddGrammarExpr(const GrammarExpr& grammar_expr) { grammar_->grammar_expr_indptr_.push_back(grammar_->grammar_expr_data_.size()); grammar_->grammar_expr_data_.push_back(static_cast(grammar_expr.type)); grammar_->grammar_expr_data_.push_back(grammar_expr.data_len); grammar_->grammar_expr_data_.insert( grammar_->grammar_expr_data_.end(), grammar_expr.data, grammar_expr.data + grammar_expr.data_len ); return static_cast(grammar_->grammar_expr_indptr_.size()) - 1; } int32_t GrammarBuilder::AddByteString(const std::vector& bytes) { return AddGrammarExpr( {GrammarExprType::kByteString, bytes.data(), static_cast(bytes.size())} ); } int32_t GrammarBuilder::AddByteString(const std::string& str) { std::vector bytes; bytes.reserve(str.size()); for (char c : str) { bytes.push_back(static_cast(static_cast(c))); } return AddGrammarExpr( {GrammarExprType::kByteString, bytes.data(), static_cast(bytes.size())} ); } int32_t GrammarBuilder::AddCharacterClass( const std::vector& elements, bool is_negative ) { std::vector data; data.reserve(1 + elements.size() * 2); data.push_back(static_cast(is_negative)); for (const auto& range : elements) { data.push_back(range.lower); data.push_back(range.upper); } return AddGrammarExpr( {GrammarExprType::kCharacterClass, data.data(), static_cast(data.size())} ); } int32_t GrammarBuilder::AddCharacterClassStar( const std::vector& elements, bool is_negative ) { std::vector data; data.reserve(1 + elements.size() * 2); data.push_back(static_cast(is_negative)); for (const auto& range : elements) { data.push_back(range.lower); data.push_back(range.upper); } return AddGrammarExpr( {GrammarExprType::kCharacterClassStar, data.data(), static_cast(data.size())} ); } int32_t GrammarBuilder::AddEmptyStr() { return AddGrammarExpr({GrammarExprType::kEmptyStr, nullptr, 0}); } int32_t GrammarBuilder::AddTokenSet(const std::vector& token_ids) { return AddGrammarExpr( {GrammarExprType::kToken, token_ids.data(), static_cast(token_ids.size())} ); } int32_t GrammarBuilder::AddExcludeTokenSet(const std::vector& token_ids) { return AddGrammarExpr( {GrammarExprType::kExcludeToken, token_ids.data(), static_cast(token_ids.size())} ); } int32_t GrammarBuilder::AddRuleRef(int32_t rule_id) { std::vector data; data.push_back(rule_id); return AddGrammarExpr({GrammarExprType::kRuleRef, data.data(), static_cast(data.size())} ); } int32_t GrammarBuilder::AddSequence(const std::vector& elements) { return AddGrammarExpr( {GrammarExprType::kSequence, elements.data(), static_cast(elements.size())} ); } int32_t GrammarBuilder::AddChoices(const std::vector& choices) { return AddGrammarExpr( {GrammarExprType::kChoices, choices.data(), static_cast(choices.size())} ); } int32_t GrammarBuilder::AddTagDispatch(const Grammar::Impl::TagDispatch& tag_dispatch) { std::vector data; data.reserve(tag_dispatch.tag_rule_pairs.size() * 2 + 2); for (const auto& [tag, rule_id] : tag_dispatch.tag_rule_pairs) { data.push_back(AddByteString(tag)); data.push_back(rule_id); } data.push_back(static_cast(tag_dispatch.loop_after_dispatch)); std::vector exclude_str_expr_ids; for (const auto& exclude_str : tag_dispatch.excludes) { exclude_str_expr_ids.push_back(AddByteString(exclude_str)); } data.push_back(AddChoices(exclude_str_expr_ids)); return AddGrammarExpr( {GrammarExprType::kTagDispatch, data.data(), static_cast(data.size())} ); } int32_t GrammarBuilder::AddTokenTagDispatch( const Grammar::Impl::TokenTagDispatch& token_tag_dispatch ) { std::vector data; data.push_back(static_cast(token_tag_dispatch.trigger_rule_pairs.size())); for (const auto& [token_id, rule_id] : token_tag_dispatch.trigger_rule_pairs) { data.push_back(token_id); data.push_back(rule_id); } data.push_back(static_cast(token_tag_dispatch.loop_after_dispatch)); data.push_back(static_cast(token_tag_dispatch.excludes.size())); for (auto token_id : token_tag_dispatch.excludes) { data.push_back(token_id); } return AddGrammarExpr( {GrammarExprType::kTokenTagDispatch, data.data(), static_cast(data.size())} ); } int32_t GrammarBuilder::AddRepeat( int32_t ref_rule_id, int32_t min_repeat_count, int32_t max_repeat_count ) { std::vector data({ref_rule_id, min_repeat_count, max_repeat_count}); return AddGrammarExpr({GrammarExprType::kRepeat, data.data(), static_cast(data.size())}); } int32_t GrammarBuilder::AddRepeatFromExpr( const std::string& cur_rule_name, int32_t grammar_expr_id, int32_t min_repeat_count, int32_t max_repeat_count ) { const auto& expr = GetGrammarExpr(grammar_expr_id); int32_t ref_rule_id; if (expr.type == GrammarExprType::kRuleRef) { ref_rule_id = expr[0]; } else { ref_rule_id = AddRule(GetNewRuleName(cur_rule_name), grammar_expr_id); } return AddRepeat(ref_rule_id, min_repeat_count, max_repeat_count); } int32_t GrammarBuilder::NumGrammarExprs() const { return grammar_->NumGrammarExprs(); } GrammarBuilder::GrammarExpr GrammarBuilder::GetGrammarExpr(int32_t grammar_expr_id) { return grammar_->GetGrammarExpr(grammar_expr_id); } int32_t GrammarBuilder::AddRule(const Rule& rule) { int32_t id = static_cast(grammar_->rules_.size()); grammar_->rules_.push_back(rule); XGRAMMAR_CHECK(rule_name_to_id_.count(rule.name) == 0); rule_name_to_id_[rule.name] = id; return id; } int32_t GrammarBuilder::AddRule(const std::string& name, int32_t body_expr_id) { return AddRule({name, body_expr_id}); } int32_t GrammarBuilder::AddRuleWithHint(const std::string& name_hint, int32_t body_expr_id) { return AddRule({GetNewRuleName(name_hint), body_expr_id}); } int32_t GrammarBuilder::NumRules() const { return grammar_->NumRules(); } const GrammarBuilder::Rule& GrammarBuilder::GetRule(int32_t rule_id) const { return grammar_->rules_[rule_id]; } int32_t GrammarBuilder::AddEmptyRule(const std::string& name) { return AddRule({name, -1}); } int32_t GrammarBuilder::AddEmptyRuleWithHint(const std::string& name_hint) { return AddRule({GetNewRuleName(name_hint), -1}); } void GrammarBuilder::UpdateRuleBody(int32_t rule_id, int32_t body_expr_id) { XGRAMMAR_CHECK(rule_id >= 0 && rule_id < static_cast(grammar_->rules_.size())) << "Rule id " << rule_id << " is out of range."; grammar_->rules_[rule_id].body_expr_id = body_expr_id; } void GrammarBuilder::UpdateRuleBody(std::string rule_name, int32_t body_expr_id) { int32_t rule_id = GetRuleId(rule_name); XGRAMMAR_CHECK(rule_id != -1) << "Rule " << rule_name << " is not found."; UpdateRuleBody(rule_id, body_expr_id); } void GrammarBuilder::UpdateLookaheadAssertion(int32_t rule_id, int32_t lookahead_assertion_id) { XGRAMMAR_CHECK(rule_id < static_cast(grammar_->rules_.size())) << "Rule id " << rule_id << " is out of range."; grammar_->rules_[rule_id].lookahead_assertion_id = lookahead_assertion_id; } void GrammarBuilder::UpdateLookaheadExact(int32_t rule_id, bool is_exact) { XGRAMMAR_CHECK(rule_id < static_cast(grammar_->rules_.size())) << "Rule id " << rule_id << " is out of range."; grammar_->rules_[rule_id].is_exact_lookahead = is_exact; } void GrammarBuilder::UpdateLookaheadAssertion( std::string rule_name, int32_t lookahead_assertion_id ) { int32_t rule_id = GetRuleId(rule_name); XGRAMMAR_CHECK(rule_id != -1) << "Rule " << rule_name << " is not found."; UpdateLookaheadAssertion(rule_id, lookahead_assertion_id); } std::string GrammarBuilder::GetNewRuleName(const std::string& name_hint) { if (rule_name_to_id_.count(name_hint) == 0) { return name_hint; } int* cnt = &next_cnt_per_hint_[name_hint]; if (*cnt == 0) { *cnt = 1; } while (rule_name_to_id_.count(name_hint + "_" + std::to_string(*cnt)) != 0) { ++(*cnt); } return name_hint + "_" + std::to_string(*cnt); } int32_t GrammarBuilder::GetRuleId(const std::string& name) const { auto it = rule_name_to_id_.find(name); if (it == rule_name_to_id_.end()) { return -1; } else { return it->second; } } } // namespace xgrammar xgrammar-0.2.3/cpp/grammar_builder.h000066400000000000000000000167521521764210300174350ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar_builder.h * \brief The header for the building the BNF AST. */ #ifndef XGRAMMAR_GRAMMAR_BUILDER_H_ #define XGRAMMAR_GRAMMAR_BUILDER_H_ #include #include #include #include #include #include "grammar_impl.h" #include "xgrammar/grammar.h" namespace xgrammar { /*! * \brief Helper class to build a BNF grammar. */ class GrammarBuilder { public: using Rule = Grammar::Impl::Rule; using GrammarExprType = Grammar::Impl::GrammarExprType; using GrammarExpr = Grammar::Impl::GrammarExpr; /*! \brief One element of a character class, containing a lower and a upper bound. Both bounds are * inclusive. */ struct CharacterClassElement { int32_t lower; int32_t upper; }; /*! \brief Default constructor. Creates a new grammar object. */ GrammarBuilder(); /*! \brief Constructor. Creates a new grammar object from an existing grammar. */ GrammarBuilder(const Grammar& grammar); /*! * \brief Get the result grammar. This function will also set the root rule to the rule with the * specified name. The rule should be already added to the grammar. * \param root_rule_name The name of the root rule. Default is "root". */ Grammar Get(const std::string& root_rule_name = "root"); /*! * \brief Get the result grammar. This function will also set the root rule to the rule with * the specified id. The rule should be already added to the grammar. * \param root_rule_id The id of the root rule. */ Grammar Get(int32_t root_rule_id); /****************** GrammarExpr handling ******************/ /*! \brief Add a grammar_expr and return the grammar_expr id. */ int32_t AddGrammarExpr(const GrammarExpr& grammar_expr); /*! * \brief Add a GrammarExpr for string stored in bytes. * \param bytes A vector of int32_t, each representing a byte (0~255) in the string. * The string is stored in int32 vector to match the storage format of the grammar. */ int32_t AddByteString(const std::vector& bytes); /*! * \brief Add a GrammarExpr for string stored in bytes. * \param str The string to be added. */ int32_t AddByteString(const std::string& str); /*! * \brief Add a GrammarExpr for a character class. * \param elements A vector of CharacterClassElement, each containing a lower and a upper bound. * \param is_negative Whether the character class is negated. */ int32_t AddCharacterClass( const std::vector& elements, bool is_negative = false ); /*! * \brief Add a GrammarExpr for a star quantifier of a character class. * \param elements A vector of CharacterClassElement, each containing a lower and a upper bound. * \param is_negative Whether the character class is negated. */ int32_t AddCharacterClassStar( const std::vector& elements, bool is_negative = false ); /*! \brief Add a GrammarExpr for empty string.*/ int32_t AddEmptyStr(); /*! \brief Add a GrammarExpr for kToken (token-level matching). */ int32_t AddTokenSet(const std::vector& token_ids); /*! \brief Add a GrammarExpr for kExcludeToken (excluded token-level matching). */ int32_t AddExcludeTokenSet(const std::vector& token_ids); /*! \brief Add a GrammarExpr for rule reference.*/ int32_t AddRuleRef(int32_t rule_id); /*! \brief Add a GrammarExpr for GrammarExpr sequence.*/ int32_t AddSequence(const std::vector& elements); /*! \brief Add a GrammarExpr for GrammarExpr choices.*/ int32_t AddChoices(const std::vector& choices); /*! * \brief Add a GrammarExpr for tag dispatch. * \param tag_dispatch_list A list of pairs of tag_expr_id and rule_id. */ int32_t AddTagDispatch(const Grammar::Impl::TagDispatch& tag_dispatch); /*! \brief Encode a TokenTagDispatch struct into a kTokenTagDispatch expr. */ int32_t AddTokenTagDispatch(const Grammar::Impl::TokenTagDispatch& token_tag_dispatch); int32_t AddRepeat(int32_t ref_rule_id, int32_t min_repeat_count, int32_t max_repeat_count); /*! * \brief Add a repeat GrammarExpr from an arbitrary grammar expression. If the expression is * not a rule reference, a new rule is created to wrap it. * \param cur_rule_name Name hint for generated rules. * \param grammar_expr_id The expression to repeat. * \param min_repeat_count Minimum repeat count (inclusive). * \param max_repeat_count Maximum repeat count (inclusive), or -1 for unbounded. */ int32_t AddRepeatFromExpr( const std::string& cur_rule_name, int32_t grammar_expr_id, int32_t min_repeat_count, int32_t max_repeat_count ); /*! \brief Get the number of grammar_exprs. */ int32_t NumGrammarExprs() const; /*! \brief Get the grammar_expr with the given id. */ GrammarExpr GetGrammarExpr(int32_t grammar_expr_id); /****************** Rule handling ******************/ /*! \brief Add a rule and return the rule id. */ int32_t AddRule(const Rule& rule); int32_t AddRule(const std::string& name, int32_t body_expr_id); int32_t AddRuleWithHint(const std::string& name_hint, int32_t body_expr_id); int32_t NumRules() const; /*! \brief Get the rule with the given id. */ const Rule& GetRule(int32_t rule_id) const; /*! * \brief Add an rule without body, and return the rule id. The rule body should be set later * with GrammarBuilder::UpdateRuleBody. This method is useful for cases where the rule id is * required to build the rule body. * \sa GrammarBuilder::UpdateRuleBody */ int32_t AddEmptyRule(const std::string& name); int32_t AddEmptyRuleWithHint(const std::string& name_hint); /*! * \brief Update the rule body of the given rule, specified by rule id. Can be used to set the * rule body of a rule inserted by GrammarBuilder::AddEmptyRule. */ void UpdateRuleBody(int32_t rule_id, int32_t body_expr_id); /*! * \brief Update the rule body of the given rule, specified by rule name. Can be used to set the * rule body of a rule inserted by GrammarBuilder::AddEmptyRule. */ void UpdateRuleBody(std::string rule_name, int32_t body_expr_id); /*! * \brief Add a lookahead assertion to a rule referred by the given rule_id. The lookahead * assertion should be a sequence GrammarExpr id. An id of -1 means no lookahead assertion. */ void UpdateLookaheadAssertion(int32_t rule_id, int32_t lookahead_assertion_id); void UpdateLookaheadExact(int32_t rule_id, bool is_exact = true); /*! * \brief Add a lookahead assertion to a rule referred by the given name. The lookahead * assertion should be a sequence GrammarExpr id. An id of -1 means no lookahead assertion. */ void UpdateLookaheadAssertion(std::string rule_name, int32_t lookahead_assertion_id); /*! * \brief Find a name for a new rule starting with the given name hint. Some integer suffix (_1, * _2, ...) may be added to avoid name conflict. */ std::string GetNewRuleName(const std::string& name_hint); /*! * \brief Get the rule id of the rule with the given name. Return -1 if not found. */ int32_t GetRuleId(const std::string& name) const; private: // Mutable pointer to the grammar object. std::shared_ptr grammar_; // Map from rule name to rule id. std::unordered_map rule_name_to_id_; // Cache of next suffix index per name_hint for GetNewRuleName. std::unordered_map next_cnt_per_hint_; }; } // namespace xgrammar #endif // XGRAMMAR_GRAMMAR_BUILDER_H_ xgrammar-0.2.3/cpp/grammar_compiler.cc000066400000000000000000001677331521764210300177650ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/compiler.cc */ #include #include #include #include #include #include #include #include #include #include #include #include #include "compiled_grammar_impl.h" #include "earley_parser.h" #include "fsm.h" #include "grammar_functor.h" #include "grammar_impl.h" #include "support/dynamic_bitset.h" #include "support/int_set.h" #include "support/logging.h" #include "support/thread_pool.h" #include "support/thread_safe_cache.h" #include "support/utils.h" #include "tokenizer_info_impl.h" #include "xgrammar/grammar.h" #include "xgrammar/tokenizer_info.h" namespace xgrammar { /************** AdaptiveTokenMaskCache Generator **************/ /*! \brief The concrete implementation of GrammarMatcherNode. */ class GrammarMatcherForTokenMaskCache : public EarleyParser { public: GrammarMatcherForTokenMaskCache( const Grammar& grammar, const ParserState& init_state, const std::unordered_map& tag_dispatch_rule_id_to_second_slicing_bitset, const TokenizerInfo& tokenizer_info, std::optional& rule_level_cache, const bool& need_expand = true ) : EarleyParser(grammar, init_state), init_rule_id_(init_state.rule_id), initial_state_(init_state), tag_dispatch_rule_id_to_second_slicing_bitset_(tag_dispatch_rule_id_to_second_slicing_bitset ), tokenizer_info_(tokenizer_info), rule_level_cache_(rule_level_cache) {} /*! * \brief Get the adaptive token mask for the given ParserState. * \param is_root_rule Whether to consider the parent rule. If false, there will be * no uncertain tokens. Useful for the root rule. */ AdaptiveTokenMask GetAdaptiveTokenMask(bool is_root_rule); /*! * \brief Get the token mask for the given ParserState. * \param first_char_mask The first character mask. * \param is_root_rule Whether to consider the parent rule. If false, there will be * no uncertain tokens. Useful for the root rule. * \returns True if the rejected indices are filled as usual, False otherwise. * It's used to determine which construction function will be used. */ bool GetTokenMaskWithFirstCharacterCheck( const std::bitset<256>& first_char_mask, bool is_root_rule, const std::vector& token_edge_accepted ); /*! * \brief Adapt the cache with lookahead assertion. * \param cache The adaptive token mask to be adapted. * \param is_root_rule Whether to consider the parent rule. */ void AdaptCacheWithLookahead(AdaptiveTokenMask* cache, bool is_root_rule); private: /*! \brief Check if a token can pass the lookahead assertion. */ std::pair IsTokenPassLookaheadAssertion( const std::string& token, const std::vector& can_reach_end_stack ); /*! * \brief Check if speculative calculation will be applied. * \return first: whether speculative calculation is applicable. * \return second: part of the first character mask, * which can be used in speculative calculation. */ std::pair> GetSpeculativeCalculation(); /*! * \brief Get the first character mask. * \param first_character_mask the bitset to store the first character mask. */ void GetFirstCharacterMask(std::bitset<256>& first_character_mask); /*! * \brief Compute sorted vocab indices accepted by token edges at the current FSM state. * Token(ids) edges accept listed token IDs. * ExcludeToken(ids) edges accept all tokens except listed IDs. * \return Sorted, deduplicated vector of accepted sorted vocab indices. */ const std::vector& GetTokenEdgeAcceptedIndices(); // The id of the initial rule. int32_t init_rule_id_; // The initial state of the parser. ParserState initial_state_; /*! * \brief This is a mapping from TagDispatch rule id to the bitset used for second slicing. * \note If a rule is a TagDispatch rule, then there will be an AC automaton for its triggers. * Which means that it can accept a lot of tokens. However, it will be slow to check a lot of * tokens. The DynamicBitset here is used to do a second slicing: if a token's substr(1, n - 1) * can be accepted by the start state of the AC automaton, then it will be True in the bitset. * When we check a token, we first check if its first character can transit to the start state. * If yes, then we check if it is in the bitset. If yes, then we accept it directly. */ const std::unordered_map& tag_dispatch_rule_id_to_second_slicing_bitset_; const TokenizerInfo& tokenizer_info_; std::optional rule_level_cache_; // Temporary data for GetAdaptiveTokenMask. std::vector tmp_accepted_indices_; std::vector tmp_rejected_indices_; std::vector tmp_uncertain_indices_; std::vector tmp_rejected_by_lookahead_indices_; std::vector tmp_accepted_by_lookahead_indices_; std::vector tmp_can_reach_end_stack_; std::vector tmp_can_reach_end_prefix_or_stack_; // Temporary data for GetTokenEdgeAcceptedIndices. std::vector tmp_token_edge_accepted_; std::vector tmp_token_edge_excluded_; }; void GrammarMatcherForTokenMaskCache::AdaptCacheWithLookahead( AdaptiveTokenMask* cache_ptr, bool is_root_rule ) { AdaptiveTokenMask& cache = *cache_ptr; const auto& sorted_decoded_vocab = tokenizer_info_.GetSortedDecodedVocab(); const auto& subtree_nodes_range = tokenizer_info_.GetTrieSubtreeNodesRange(); const std::string* prev_token = nullptr; bool is_exact_lookahead = grammar_->GetRule(init_rule_id_).is_exact_lookahead; int prev_matched_size = 0; int last_rejected_range = 0; int last_uncertain_range = 0; if (is_root_rule) { tmp_rejected_indices_ = cache.uncertain_indices; } else { const auto& lookahead_id = grammar_->GetRule(init_rule_id_).lookahead_assertion_id; if (lookahead_id == -1) { return; } for (const auto& uncertain_index : cache.uncertain_indices) { const auto& token = sorted_decoded_vocab[uncertain_index].second; // Many tokens may contain the same prefix, so we will avoid unnecessary matching // by finding the longest common prefix with the previous token. bool accepted = true; if (uncertain_index < last_rejected_range) { tmp_rejected_indices_.push_back(uncertain_index); continue; } if (uncertain_index < last_uncertain_range) { // This token is already marked as uncertain. continue; } if (prev_token != nullptr) { int lcp_len = std::mismatch(token.begin(), token.end(), prev_token->begin(), prev_token->end()) .first - token.begin(); if (lcp_len > prev_matched_size) { // Case 1. The common prefix is rejected by the matcher in the last token. Reject // directly. accepted = false; } else if (lcp_len < prev_matched_size) { // Case 2. The common prefix is shorter than the previous matched size. Rollback // the non-common part. PopLastStates(prev_matched_size - lcp_len); tmp_can_reach_end_stack_.erase( tmp_can_reach_end_stack_.end() - (prev_matched_size - lcp_len), tmp_can_reach_end_stack_.end() ); tmp_can_reach_end_prefix_or_stack_.erase( tmp_can_reach_end_prefix_or_stack_.end() - (prev_matched_size - lcp_len), tmp_can_reach_end_prefix_or_stack_.end() ); } prev_matched_size = std::min(prev_matched_size, lcp_len); } prev_token = &token; if (accepted) { // Accept the rest chars one by one. for (int j = prev_matched_size; j < static_cast(token.size()); ++j) { if (!Advance(token[j])) { accepted = false; break; } tmp_can_reach_end_stack_.push_back(IsCompleted()); tmp_can_reach_end_prefix_or_stack_.push_back( tmp_can_reach_end_stack_.back() || tmp_can_reach_end_prefix_or_stack_.back() ); prev_matched_size = j + 1; } } XGRAMMAR_DCHECK(!tmp_can_reach_end_prefix_or_stack_.empty()); bool can_reach_end = tmp_can_reach_end_prefix_or_stack_.back(); XGRAMMAR_DCHECK(!accepted) << "All the tokens are at least uncertain!"; if (can_reach_end && prev_matched_size > 0) { auto [lookahead_accepted, lookahead_completed] = IsTokenPassLookaheadAssertion(token, tmp_can_reach_end_stack_); if ((!is_root_rule) && lookahead_accepted) { if (lookahead_completed || !is_exact_lookahead) { tmp_uncertain_indices_.push_back(uncertain_index); } else { tmp_accepted_indices_.push_back(uncertain_index); } } else { tmp_rejected_indices_.push_back(uncertain_index); last_rejected_range = subtree_nodes_range[uncertain_index]; } } else { tmp_rejected_indices_.push_back(uncertain_index); last_rejected_range = subtree_nodes_range[uncertain_index]; } } } // This strategy ensures the consistency of the cache storage type in most cases. // However, in this case, the storage type is inconsistent: // 1. The original cache is accepted_indices, and rejected_indices is also small. // After adapting with lookahead, |accepted_indices| + |accepted_by_lookahead_indices| > // |rejected_indices| + |rejected_by_lookahead_indices|, and |rejected_indices| + // |rejected_by_lookahead_indices| < AdaptiveTokenMask::USE_BITSET_THRESHOLD. In this case, it // should be kRejected, but ignored. // 2. The original cache is rejected_indices, and accepted_indices is also small. // After adapting with lookahead, |accepted_indices| + |accepted_by_lookahead_indices| < // |rejected_indices| + |rejected_by_lookahead_indices|, and |accepted_indices| + // |accepted_by_lookahead_indices| < AdaptiveTokenMask::USE_BITSET_THRESHOLD. In this case, it // should be kAccepted, but ignored. These two cases are very rare in practice, and the impact is // very limited, so we ignore them for simplicity. cache.uncertain_indices = tmp_uncertain_indices_; switch (cache.store_type) { case AdaptiveTokenMask::StoreType::kAccepted: { if (cache.accepted_indices.size() + tmp_accepted_indices_.size() < AdaptiveTokenMask::USE_BITSET_THRESHOLD) { IntsetUnion(&cache.accepted_indices, tmp_accepted_indices_); break; } // Transform to bitset. cache.store_type = AdaptiveTokenMask::StoreType::kAcceptedBitset; cache.accepted_bitset = DynamicBitset(tokenizer_info_.GetVocabSize()); for (const auto& accepted_index : cache.accepted_indices) { cache.accepted_bitset.Set(sorted_decoded_vocab[accepted_index].first); } for (const auto& accepted_index : tmp_accepted_indices_) { cache.accepted_bitset.Set(sorted_decoded_vocab[accepted_index].first); } cache.accepted_indices.clear(); break; } case AdaptiveTokenMask::StoreType::kRejected: { if (cache.rejected_indices.size() + tmp_rejected_indices_.size() < AdaptiveTokenMask::USE_BITSET_THRESHOLD) { IntsetUnion(&cache.rejected_indices, tmp_rejected_indices_); break; } // Transform to bitset. cache.store_type = AdaptiveTokenMask::StoreType::kAcceptedBitset; cache.accepted_bitset = DynamicBitset(tokenizer_info_.GetVocabSize()); cache.accepted_bitset.Set(); for (const auto& special_index : tokenizer_info_.GetSpecialTokenIds()) { cache.accepted_bitset.Reset(special_index); } for (const auto& uncertain_index : cache.uncertain_indices) { cache.accepted_bitset.Reset(sorted_decoded_vocab[uncertain_index].first); } for (const auto& rejected_index : cache.rejected_indices) { cache.accepted_bitset.Reset(sorted_decoded_vocab[rejected_index].first); } for (const auto& rejected_index : tmp_rejected_indices_) { cache.accepted_bitset.Reset(sorted_decoded_vocab[rejected_index].first); } cache.rejected_indices.clear(); break; } case AdaptiveTokenMask::StoreType::kAcceptedBitset: { for (const auto& accepted_index : tmp_accepted_indices_) { cache.accepted_bitset.Set(sorted_decoded_vocab[accepted_index].first); } break; } } } std::pair GrammarMatcherForTokenMaskCache::IsTokenPassLookaheadAssertion( const std::string& token, const std::vector& can_reach_end_stack ) { bool accepted = true; bool can_reach_end = true; auto lookahead_assertion_id = grammar_->GetRule(init_rule_id_).lookahead_assertion_id; if (lookahead_assertion_id == -1) { return {accepted, can_reach_end}; } auto lookahead_state = ParserState(/*rule_id*/ -1, lookahead_assertion_id, 0, ParserState::kNoPrevInputPos, 0); PushStateAndExpand(lookahead_state); int token_len = token.size(); if (IsCompleted()) { // If the lookahead assertion is already completed, we can accept the token. PopLastStates(1); return {accepted, can_reach_end}; } // Find all positions that can come to and end. Then check if the suffix from that position // can be accepted by the lookahead assertion. for (int i = static_cast(can_reach_end_stack.size()) - 1; i >= 0; --i) { if (!can_reach_end_stack[i]) { continue; } int last_accept_pos = i - 1; for (int pos = i; pos < token_len; ++pos) { if (!Advance(token[pos])) { break; } last_accept_pos = pos; // Case 1. The whole rule is finished. if (IsCompleted()) { // accepted chars: pos - i + 1 // we need to rollback the pushed initial state as well PopLastStates(pos - i + 2); return {accepted, can_reach_end}; } } // Case 2. The whole token is accepted if (last_accept_pos == token_len - 1) { PopLastStates(last_accept_pos - i + 2); can_reach_end = false; return {accepted, can_reach_end}; } // Case 3. The token is not accepted. Check the next position. PopLastStates(last_accept_pos - i + 1); } PopLastStates(1); can_reach_end = false; accepted = false; return {accepted, can_reach_end}; } // Comparator for std::pair based on the string value. class IntStringPairComparator { public: bool operator()( const std::pair& lhs, const std::pair& rhs ) const { return lhs.second < rhs.second; } }; int GetPossibleTokenIntervals( const std::vector>& sorted_decoded_vocab, const std::bitset<256>& first_char_mask, std::vector>& possible_intervals ) { int possible_token_num = 0; int matched_size = 0; int last_interval_end = -1; for (int32_t i = 0; i < 256; i++) { if (first_char_mask[i]) { if (last_interval_end == -1) { last_interval_end = i; } } else { if (last_interval_end != -1) { int32_t interval_left_end = std::lower_bound( sorted_decoded_vocab.begin() + matched_size, sorted_decoded_vocab.end(), std::make_pair(0, std::string(1, static_cast(last_interval_end))), IntStringPairComparator() ) - sorted_decoded_vocab.begin(); int32_t interval_right_end = std::lower_bound( sorted_decoded_vocab.begin() + interval_left_end, sorted_decoded_vocab.end(), std::make_pair(0, std::string(1, static_cast(i))), IntStringPairComparator() ) - sorted_decoded_vocab.begin(); possible_intervals.emplace_back(interval_left_end, interval_right_end); possible_token_num += interval_right_end - interval_left_end; last_interval_end = -1; matched_size = interval_right_end; } } } if (last_interval_end != -1) { // If the last interval is not closed, we need to close it. int32_t interval_left_end = std::lower_bound( sorted_decoded_vocab.begin() + matched_size, sorted_decoded_vocab.end(), std::make_pair(0, std::string(1, static_cast(last_interval_end))), IntStringPairComparator() ) - sorted_decoded_vocab.begin(); possible_intervals.emplace_back(interval_left_end, sorted_decoded_vocab.size()); possible_token_num += sorted_decoded_vocab.size() - interval_left_end; } return possible_token_num; } std::pair> GrammarMatcherForTokenMaskCache::GetSpeculativeCalculation() { using GrammarExprType = Grammar::Impl::GrammarExprType; // If the initial rule is a tag dispatch, we will check if it can achieve its initial state. const auto& rule = grammar_->GetRule(init_rule_id_); const auto& rule_body = grammar_->GetGrammarExpr(rule.body_expr_id); if (rule_body.type == GrammarExprType::kTagDispatch) { std::bitset<256> speculative_mask; XGRAMMAR_DCHECK(grammar_->per_rule_fsms[init_rule_id_].has_value()); const auto& fsm = grammar_->per_rule_fsms[init_rule_id_].value(); for (const auto& edge : fsm.GetFsm().GetFsm().GetEdges(initial_state_.element_id)) { if (edge.target != fsm.GetFsm().GetStart()) { continue; } if (!edge.IsCharRange()) { continue; } for (int32_t ch = edge.min; ch <= edge.max; ++ch) { speculative_mask.set(ch); } } return {true, speculative_mask}; } // Check if the initial state is self-recursive-like via FSM. XGRAMMAR_DCHECK(grammar_->per_rule_fsms[init_rule_id_].has_value()); bool can_be_applied = false; std::bitset<256> speculative_mask; const auto& fsm = grammar_->per_rule_fsms[init_rule_id_].value(); XGRAMMAR_DCHECK(initial_state_.element_id < fsm.GetFsm().NumStates()) << "Initial State's element id cannot exceed the whole FSM's number of states."; for (const auto& edge : fsm.GetFsm().GetFsm().GetEdges(initial_state_.element_id)) { if (edge.IsCharRange()) { // Case A: The edge is towards itself. if (edge.target == initial_state_.element_id) { can_be_applied = true; for (int ch = edge.min; ch <= edge.max; ++ch) { speculative_mask.set(ch); } continue; } // Case B: The state is the start state, and there's an edge to another state, // which calls the fsm itself. if (fsm.GetFsm().GetStart() == initial_state_.element_id) { for (const auto& next_edge : fsm.GetFsm().GetFsm().GetEdges(edge.target)) { if ((next_edge.IsRuleRef() && next_edge.GetRefRuleId() == init_rule_id_) || (next_edge.IsRepeatRef() && fsm.GetFsm().GetFsm().GetRepeatEdgeInfo(next_edge.GetAuxIndex()).RuleId() == init_rule_id_)) { can_be_applied = true; for (int ch = edge.min; ch <= edge.max; ++ch) { speculative_mask.set(ch); } break; } } } } } return {can_be_applied, speculative_mask}; } bool GrammarMatcherForTokenMaskCache::GetTokenMaskWithFirstCharacterCheck( const std::bitset<256>& first_char_mask, bool is_root_rule, const std::vector& token_edge_accepted ) { const auto& sorted_decoded_vocab = tokenizer_info_.GetSortedDecodedVocab(); const auto& subtree_nodes_range = tokenizer_info_.GetTrieSubtreeNodesRange(); // the pair (a, b) means [a, b). Intialize the possible intervals. std::vector> possible_intervals; int possible_token_num = GetPossibleTokenIntervals(sorted_decoded_vocab, first_char_mask, possible_intervals); // Check if the type of the mask can be rejected. tmp_accepted_indices_.reserve(possible_token_num); bool fill_reject_indices = (sorted_decoded_vocab.size() - possible_token_num) < AdaptiveTokenMask::USE_BITSET_THRESHOLD; XGRAMMAR_DCHECK(possible_intervals.size() > 0) << "There should be at least one possible interval for the first character mask."; if (possible_intervals[0].first != 0 && fill_reject_indices) { for (int i = 0; i < possible_intervals[0].first; ++i) { tmp_rejected_indices_.push_back(i); } } XGRAMMAR_DCHECK(init_rule_id_ != -1 && grammar_->per_rule_fsms[init_rule_id_].has_value()); auto [speculative_calculation, speculative_mask] = GetSpeculativeCalculation(); int prev_matched_size = 0; int last_rejected_range = 0; const bool& is_exact_lookahead = grammar_->GetRule(init_rule_id_).is_exact_lookahead; std::optional definite_accepted_bitset = std::nullopt; const bool is_tag_dispatch_rule = grammar_->GetGrammarExpr(grammar_->GetRule(init_rule_id_).body_expr_id).type == Grammar::Impl::GrammarExprType::kTagDispatch; if (is_tag_dispatch_rule) { XGRAMMAR_DCHECK(tag_dispatch_rule_id_to_second_slicing_bitset_.count(init_rule_id_) > 0); definite_accepted_bitset = &tag_dispatch_rule_id_to_second_slicing_bitset_.at(init_rule_id_); } const std::string* prev_token = nullptr; int32_t skip_ptr = 0; const int32_t skip_size = static_cast(token_edge_accepted.size()); for (size_t interval_idx = 0; interval_idx < possible_intervals.size(); ++interval_idx) { const auto& interval = possible_intervals[interval_idx]; for (int i = interval.first; i < interval.second; ++i) { // Skip tokens already accepted by token edges (avoid expensive Earley simulation). while (skip_ptr < skip_size && token_edge_accepted[skip_ptr] < i) ++skip_ptr; if (skip_ptr < skip_size && token_edge_accepted[skip_ptr] == i) continue; // Check if the current token is in the rejected range. i.e. check if the current token // is on the subtree of the rejected token. if (i < last_rejected_range) { if (fill_reject_indices) { tmp_rejected_indices_.push_back(i); fill_reject_indices = tmp_rejected_indices_.size() >= AdaptiveTokenMask::USE_BITSET_THRESHOLD ? false : fill_reject_indices; } else { i = last_rejected_range - 1; } continue; } const auto& token = sorted_decoded_vocab[i].second; // This optimization is useful for simple self-recursive rules, like string content. if (speculative_calculation) { // Optimization for tag dispatch rules. if (definite_accepted_bitset.has_value()) { // If the token is empty, it must be accepted. if (token.empty()) { tmp_accepted_indices_.push_back(i); continue; } // If the token doesn't contain tags or stop strings since the second character, and it // will transit to the start state after consuming the first character, it must be // accepted. if (speculative_mask[static_cast(token[0])] && (*definite_accepted_bitset.value())[i]) { tmp_accepted_indices_.push_back(i); continue; } } else { bool all_accepted = true; for (char ch : token) { // If the first character is not the ascii character or can't be accepted by the // first character mask, we need to check them in the parser. if (isascii(ch) == 0 || !speculative_mask[static_cast(ch)]) { all_accepted = false; break; } } if (all_accepted) { tmp_accepted_indices_.push_back(i); continue; } } } // Many tokens may contain the same prefix, so we will avoid unnecessary matching // by finding the longest common prefix with the previous token. bool accepted = true; if (prev_token != nullptr) { int lcp_len = std::mismatch(token.begin(), token.end(), prev_token->begin(), prev_token->end()) .first - token.begin(); if (lcp_len > prev_matched_size) { // Case 1. The common prefix is rejected by the matcher in the last token. Reject // directly. accepted = false; } else if (lcp_len < prev_matched_size) { // Case 2. The common prefix is shorter than the previous matched size. Rollback // the non-common part. PopLastStates(prev_matched_size - lcp_len); tmp_can_reach_end_stack_.erase( tmp_can_reach_end_stack_.end() - (prev_matched_size - lcp_len), tmp_can_reach_end_stack_.end() ); tmp_can_reach_end_prefix_or_stack_.erase( tmp_can_reach_end_prefix_or_stack_.end() - (prev_matched_size - lcp_len), tmp_can_reach_end_prefix_or_stack_.end() ); } prev_matched_size = std::min(prev_matched_size, lcp_len); } prev_token = &token; if (accepted) { // Accept the rest chars one by one. for (int j = prev_matched_size; j < static_cast(token.size()); ++j) { if (!Advance(token[j])) { accepted = false; break; } tmp_can_reach_end_stack_.push_back(IsCompleted()); tmp_can_reach_end_prefix_or_stack_.push_back( tmp_can_reach_end_stack_.back() || tmp_can_reach_end_prefix_or_stack_.back() ); prev_matched_size = j + 1; } } bool can_reach_end = tmp_can_reach_end_prefix_or_stack_.back(); if (accepted) { tmp_accepted_indices_.push_back(i); } else if (can_reach_end && prev_matched_size > 0) { auto [lookahead_accepted, lookahead_completed] = IsTokenPassLookaheadAssertion(token, tmp_can_reach_end_stack_); if ((!is_root_rule) && lookahead_accepted) { if (lookahead_completed || !is_exact_lookahead) { tmp_uncertain_indices_.push_back(i); } else { tmp_accepted_indices_.push_back(i); tmp_accepted_by_lookahead_indices_.push_back(i); } } else { for (int j = i; j < subtree_nodes_range[i]; j++) { tmp_rejected_indices_.push_back(j); tmp_rejected_by_lookahead_indices_.push_back(j); } i = subtree_nodes_range[i] - 1; // Skip the subtree nodes. } } else { tmp_rejected_indices_.push_back(i); last_rejected_range = subtree_nodes_range[i]; fill_reject_indices = tmp_rejected_indices_.size() >= AdaptiveTokenMask::USE_BITSET_THRESHOLD ? false : fill_reject_indices; } } if (interval_idx != possible_intervals.size() - 1 && fill_reject_indices) { const auto& next_interval = possible_intervals[interval_idx + 1]; for (int i = interval.second; i < next_interval.first; ++i) { tmp_rejected_indices_.push_back(i); } fill_reject_indices = tmp_rejected_indices_.size() >= AdaptiveTokenMask::USE_BITSET_THRESHOLD ? false : fill_reject_indices; } } // Rollback the last matched part. PopLastStates(prev_matched_size); if (possible_intervals.back().second != static_cast(sorted_decoded_vocab.size()) && fill_reject_indices) { // If the last interval is not closed, we need to reject the rest tokens. for (int i = possible_intervals.back().second; i < static_cast(sorted_decoded_vocab.size()); ++i) { tmp_rejected_indices_.push_back(i); } } return fill_reject_indices; } void GrammarMatcherForTokenMaskCache::GetFirstCharacterMask(std::bitset<256>& first_character_mask ) { first_character_mask.reset(); XGRAMMAR_DCHECK(grammar_->per_rule_fsms[init_rule_id_].has_value()); const auto& fsm = grammar_->per_rule_fsms[init_rule_id_].value(); const auto& edges = fsm.GetFsm().GetFsm().GetEdges(initial_state_.element_id); for (const auto& edge : edges) { if (edge.IsCharRange()) { for (int c = edge.min; c <= edge.max; ++c) { first_character_mask[c] = true; } } } } const std::vector& GrammarMatcherForTokenMaskCache::GetTokenEdgeAcceptedIndices() { // Compute sorted vocab indices accepted by Token(ids) and ExcludeToken(ids) edges. // Result is stored in tmp_token_edge_accepted_. tmp_token_edge_accepted_.clear(); tmp_token_edge_excluded_.clear(); XGRAMMAR_DCHECK(grammar_->per_rule_fsms[init_rule_id_].has_value()); const auto& fsm = grammar_->per_rule_fsms[init_rule_id_].value(); const auto& edges = fsm.GetFsm().GetFsm().GetEdges(initial_state_.element_id); const auto& sorted_decoded_vocab = tokenizer_info_.GetSortedDecodedVocab(); int32_t sorted_size = static_cast(sorted_decoded_vocab.size()); const auto& tid_to_sorted = tokenizer_info_.ImplPtr()->GetTokenIdToSortedVocabIndex(); bool has_exclude_token = false; for (const auto& edge : edges) { if (edge.IsToken()) { auto info = fsm.GetFsm().GetFsm().GetTokenEdgeInfo(edge.GetAuxIndex()); for (int32_t i = 0; i < info.Count(); ++i) { int32_t tid = info.TokenIds()[i]; XGRAMMAR_DCHECK(tid >= 0 && tid < static_cast(tid_to_sorted.size())); if (tid_to_sorted[tid] >= 0) { tmp_token_edge_accepted_.push_back(tid_to_sorted[tid]); } } } else if (edge.IsExcludeToken()) { has_exclude_token = true; auto info = fsm.GetFsm().GetFsm().GetExcludeTokenEdgeInfo(edge.GetAuxIndex()); for (int32_t i = 0; i < info.Count(); ++i) { int32_t tid = info.TokenIds()[i]; XGRAMMAR_DCHECK(tid >= 0 && tid < static_cast(tid_to_sorted.size())); if (tid_to_sorted[tid] >= 0) { tmp_token_edge_excluded_.push_back(tid_to_sorted[tid]); } } } } // Token-only: result = token_accepted if (!has_exclude_token) { if (!tmp_token_edge_accepted_.empty()) { std::sort(tmp_token_edge_accepted_.begin(), tmp_token_edge_accepted_.end()); tmp_token_edge_accepted_.erase( std::unique(tmp_token_edge_accepted_.begin(), tmp_token_edge_accepted_.end()), tmp_token_edge_accepted_.end() ); } return tmp_token_edge_accepted_; } // ExcludeToken: result = [0, sorted_size) - (excluded - token_accepted) // Token(ids) overrides ExcludeToken(ids) when both present. if (!tmp_token_edge_accepted_.empty()) { std::sort(tmp_token_edge_accepted_.begin(), tmp_token_edge_accepted_.end()); tmp_token_edge_accepted_.erase( std::unique(tmp_token_edge_accepted_.begin(), tmp_token_edge_accepted_.end()), tmp_token_edge_accepted_.end() ); } std::sort(tmp_token_edge_excluded_.begin(), tmp_token_edge_excluded_.end()); tmp_token_edge_excluded_.erase( std::unique(tmp_token_edge_excluded_.begin(), tmp_token_edge_excluded_.end()), tmp_token_edge_excluded_.end() ); IntsetDifference(&tmp_token_edge_excluded_, tmp_token_edge_accepted_); IntsetComplement(&tmp_token_edge_accepted_, sorted_size, tmp_token_edge_excluded_); return tmp_token_edge_accepted_; } AdaptiveTokenMask GrammarMatcherForTokenMaskCache::GetAdaptiveTokenMask(bool is_root_rule) { tmp_accepted_indices_.clear(); tmp_rejected_indices_.clear(); tmp_uncertain_indices_.clear(); tmp_rejected_by_lookahead_indices_.clear(); tmp_accepted_by_lookahead_indices_.clear(); tmp_can_reach_end_prefix_or_stack_.clear(); tmp_can_reach_end_stack_.clear(); // For every character in the current token, stores whether it is possible to reach the end of // the rule when matching until this character. Store it in a stack for later rollback. tmp_can_reach_end_stack_.push_back(false); tmp_can_reach_end_prefix_or_stack_.push_back(false); // Try to get the crossing cache. bool rule_level_cache_is_available = rule_level_cache_.has_value() && grammar_->per_rule_fsm_hashes[init_rule_id_].has_value(); std::optional fsm_hash = std::nullopt; int32_t new_state_id = -1; std::optional crossing_cache = std::nullopt; int lookahead_id = grammar_->GetRule(initial_state_.rule_id).lookahead_assertion_id; bool is_exact_lookahead = grammar_->GetRule(initial_state_.rule_id).is_exact_lookahead; std::optional lookahead_hash = std::nullopt; if (rule_level_cache_is_available) { lookahead_hash = GrammarFSMHasher::HashSequence(grammar_, lookahead_id); const auto& original_to_new_id = grammar_->per_rule_fsm_new_state_ids[init_rule_id_]; fsm_hash = grammar_->per_rule_fsm_hashes[init_rule_id_].value(); for (const auto& original_new_pair : original_to_new_id) { if (original_new_pair.first == initial_state_.element_id) { new_state_id = original_new_pair.second; break; } } XGRAMMAR_DCHECK(new_state_id != -1); const auto& fsm = grammar_->per_rule_fsms[init_rule_id_].value(); if (lookahead_hash.has_value()) { crossing_cache = rule_level_cache_->GetCache( HashCombine(fsm_hash.value(), lookahead_hash.value(), is_exact_lookahead), new_state_id, fsm.GetNodeNum(), fsm.GetEdgeNum() ); if (crossing_cache.has_value()) { // A perfect match. return crossing_cache.value(); } } crossing_cache = rule_level_cache_->GetCache( fsm_hash.value(), new_state_id, fsm.GetNodeNum(), fsm.GetEdgeNum() ); // If the rule doesn't have a lookahead, then it is exactly the same fsm. if (crossing_cache.has_value()) { AdaptCacheWithLookahead(&crossing_cache.value(), is_root_rule); return std::move(crossing_cache.value()); } } std::bitset<256> first_character_mask; GetFirstCharacterMask(first_character_mask); // Token edge accepted indices (for byte path skip + merge). const auto& token_edge_accepted = GetTokenEdgeAcceptedIndices(); // Byte path: skip tokens already accepted by token edges. bool rejected_filled; if (first_character_mask.none()) { rejected_filled = false; } else { rejected_filled = GetTokenMaskWithFirstCharacterCheck( first_character_mask, is_root_rule, token_edge_accepted ); } // Merge: token edge accepted overrides byte path classification. // accepted = accepted + token_edge_accepted // rejected = rejected - token_edge_accepted // uncertain = uncertain - token_edge_accepted if (!token_edge_accepted.empty()) { IntsetUnion(&tmp_accepted_indices_, token_edge_accepted); IntsetDifference(&tmp_rejected_indices_, token_edge_accepted); IntsetDifference(&tmp_uncertain_indices_, token_edge_accepted); } if (rejected_filled) { auto return_value = AdaptiveTokenMask( tokenizer_info_.GetVocabSize(), tokenizer_info_.GetSortedDecodedVocab(), tmp_accepted_indices_, tmp_rejected_indices_, tmp_uncertain_indices_ ); if (rule_level_cache_is_available) { if (lookahead_id == -1 && !is_root_rule) { // If the rule doesn't have a lookahead, then it is exactly the same fsm. auto& fsm = grammar_->per_rule_fsms[init_rule_id_].value(); rule_level_cache_->AddCache( fsm_hash.value(), new_state_id, fsm.GetNodeNum(), fsm.GetEdgeNum(), return_value ); return return_value; } // We can add a cache for basic fsm, and a better one for lookahead. // All the tokens rejected by lookahead should be uncertain. IntsetUnion(&tmp_uncertain_indices_, tmp_rejected_by_lookahead_indices_); IntsetUnion(&tmp_uncertain_indices_, tmp_accepted_by_lookahead_indices_); std::vector rejected_indices_without_lookahead; std::vector accepted_indices_without_lookahead; rejected_indices_without_lookahead.reserve( tmp_rejected_indices_.size() - tmp_rejected_by_lookahead_indices_.size() ); accepted_indices_without_lookahead.reserve( tmp_accepted_indices_.size() - tmp_accepted_by_lookahead_indices_.size() ); std::set_difference( tmp_rejected_indices_.begin(), tmp_rejected_indices_.end(), tmp_rejected_by_lookahead_indices_.begin(), tmp_rejected_by_lookahead_indices_.end(), std::back_inserter(rejected_indices_without_lookahead) ); std::set_difference( tmp_accepted_indices_.begin(), tmp_accepted_indices_.end(), tmp_accepted_by_lookahead_indices_.begin(), tmp_accepted_by_lookahead_indices_.end(), std::back_inserter(accepted_indices_without_lookahead) ); auto& fsm = grammar_->per_rule_fsms[init_rule_id_].value(); rule_level_cache_->AddCache( fsm_hash.value(), new_state_id, fsm.GetNodeNum(), fsm.GetEdgeNum(), AdaptiveTokenMask( tokenizer_info_.GetVocabSize(), tokenizer_info_.GetSortedDecodedVocab(), accepted_indices_without_lookahead, rejected_indices_without_lookahead, tmp_uncertain_indices_ ) ); if (lookahead_hash.has_value()) { auto& fsm = grammar_->per_rule_fsms[init_rule_id_].value(); rule_level_cache_->AddCache( HashCombine(fsm_hash.value(), lookahead_hash.value(), is_exact_lookahead), new_state_id, fsm.GetNodeNum(), fsm.GetEdgeNum(), return_value ); } } return return_value; } else { auto return_value = AdaptiveTokenMask( tokenizer_info_.GetVocabSize(), tokenizer_info_.GetSortedDecodedVocab(), tmp_accepted_indices_, tmp_uncertain_indices_ ); if (rule_level_cache_is_available) { // Prepare for cache. auto& fsm = grammar_->per_rule_fsms[init_rule_id_].value(); if (lookahead_id == -1 && !is_root_rule) { // If the rule doesn't have a lookahead, then it is exactly the same fsm. rule_level_cache_->AddCache( fsm_hash.value(), new_state_id, fsm.GetNodeNum(), fsm.GetEdgeNum(), return_value ); return return_value; } // Add 2 caches. IntsetUnion(&tmp_uncertain_indices_, tmp_rejected_by_lookahead_indices_); IntsetUnion(&tmp_uncertain_indices_, tmp_accepted_by_lookahead_indices_); std::vector accepted_indices_without_lookahead; accepted_indices_without_lookahead.reserve( tmp_accepted_indices_.size() - tmp_accepted_by_lookahead_indices_.size() ); std::set_difference( tmp_accepted_indices_.begin(), tmp_accepted_indices_.end(), tmp_accepted_by_lookahead_indices_.begin(), tmp_accepted_by_lookahead_indices_.end(), std::back_inserter(accepted_indices_without_lookahead) ); rule_level_cache_->AddCache( fsm_hash.value(), new_state_id, fsm.GetNodeNum(), fsm.GetEdgeNum(), AdaptiveTokenMask( tokenizer_info_.GetVocabSize(), tokenizer_info_.GetSortedDecodedVocab(), accepted_indices_without_lookahead, tmp_uncertain_indices_ ) ); if (lookahead_hash.has_value()) { rule_level_cache_->AddCache( HashCombine(fsm_hash.value(), lookahead_hash.value(), is_exact_lookahead), new_state_id, fsm.GetNodeNum(), fsm.GetEdgeNum(), return_value ); } } return return_value; } } /******************* GrammarCompilerNoCache *******************/ /*! * \brief The base class for the grammar compiler. Handles the compilation logic without cache. */ class GrammarCompilerSub { public: GrammarCompilerSub( const TokenizerInfo& tokenizer_info, int max_threads, std::optional rule_level_cache ) : tokenizer_info_(tokenizer_info), max_threads_(max_threads), rule_level_cache_(rule_level_cache) {} CompiledGrammar CompileBuiltinJSONGrammar(); CompiledGrammar CompileJSONSchema( const std::string& schema, bool any_whitespace, std::optional indent, std::optional> separators, bool strict_mode, std::optional max_whitespace_cnt, bool any_order ); CompiledGrammar CompileRegex(const std::string& regex); CompiledGrammar CompileStructuralTag(const std::string& structural_tag_json); CompiledGrammar CompileGrammar(const Grammar& grammar); CompiledGrammar CompileGrammar(const std::string& ebnf_str, std::string root_rule_name); private: /*! \brief The main logic. Compile the grammar with multi-threading. */ CompiledGrammar MultiThreadCompileGrammar(Grammar grammar); /*! \brief Optimization for TagDispatch. * \param compiled_grammar_impl the compiled_grammar to be optimized. * \param tag_dispatch_rule_id_to_second_slicing_bitset Return value. Mapping from the rule_id to * the definite accepted token mask. */ void TagDispatchOptimization( std::shared_ptr compiled_grammar_impl, std::unordered_map* tag_dispatch_rule_id_to_second_slicing_bitset ); /*! \brief The vocabulary associated with this storage class. */ const TokenizerInfo tokenizer_info_; /*! \brief The maximum number of threads to use. */ const int max_threads_; /*! \brief The manager of the rule level cache.*/ std::optional rule_level_cache_; }; CompiledGrammar GrammarCompilerSub::MultiThreadCompileGrammar(Grammar grammar_unoptimized) { auto compiled_grammar_impl = std::make_shared(); compiled_grammar_impl->grammar = GrammarOptimizer::Apply(grammar_unoptimized); compiled_grammar_impl->tokenizer_info = tokenizer_info_; if (tokenizer_info_.GetVocabSize() == 0) { return CompiledGrammar(compiled_grammar_impl); } std::unordered_map tag_dispatch_rule_id_to_second_slicing_bitset; TagDispatchOptimization(compiled_grammar_impl, &tag_dispatch_rule_id_to_second_slicing_bitset); // If the compiler is cache-enabled, then we hash the grammars for crossing-grammar caching. if (rule_level_cache_.has_value()) { GrammarFSMHasher().Apply(&compiled_grammar_impl->grammar); } // Step 3. Compute the adaptive token mask cache // The token mask cache is computed for these positions in the grammar: // 1. All character class or character class star (with last_utf8_bytes=0, 1, 2, 3) // 2. All byte strings (with element_in_string=0, 1, 2, ...) // since other positions will be expanded to the above positions // TODO(Charlie): Figure out how to support ThreadPool and std::mutex in WebAssembly. // Only declare ThreadPool and mutex if max_threads > 1, so when max_threads = 1, we do // not need ThreadPool or std::mutex, which throws error in runtime in WebAssembly. std::optional thread_pool; std::optional adaptive_token_mask_cache_mutex; if (max_threads_ > 1) { thread_pool.emplace(max_threads_); adaptive_token_mask_cache_mutex.emplace(); } auto add_adaptive_token_mask = [&](const ParserState& state, bool is_root_rule) { auto grammar_matcher = GrammarMatcherForTokenMaskCache( compiled_grammar_impl->grammar, state, tag_dispatch_rule_id_to_second_slicing_bitset, tokenizer_info_, rule_level_cache_, false ); auto cur_adaptive_token_mask_cache = grammar_matcher.GetAdaptiveTokenMask(is_root_rule); if (max_threads_ > 1) { std::lock_guard lock(adaptive_token_mask_cache_mutex.value()); compiled_grammar_impl->adaptive_token_mask_cache[state] = cur_adaptive_token_mask_cache; } else { compiled_grammar_impl->adaptive_token_mask_cache[state] = cur_adaptive_token_mask_cache; } }; auto add_task_adaptive_token_mask = [&](const ParserState& state, bool is_root_rule) { // Execute depending on whether we use thread_pool if (max_threads_ > 1) { thread_pool->Execute([add_adaptive_token_mask, state, is_root_rule]() { add_adaptive_token_mask(state, is_root_rule); }); } else { add_adaptive_token_mask(state, is_root_rule); } }; auto root_rule_id = compiled_grammar_impl->grammar->GetRootRuleId(); for (int32_t rule_id = 0; rule_id < static_cast(compiled_grammar_impl->grammar->NumRules()); ++rule_id) { auto rule = compiled_grammar_impl->grammar->GetRule(rule_id); const auto& rule_fsm = compiled_grammar_impl->grammar->per_rule_fsms[rule_id]; XGRAMMAR_DCHECK(rule_fsm.has_value()); auto cur_stack_element = ParserState(rule_id, rule.body_expr_id, 0, ParserState::kNoPrevInputPos, 0); std::unordered_set reachable_states; rule_fsm->GetFsm().GetReachableStates(&reachable_states); for (int i : reachable_states) { cur_stack_element.element_id = i; if (!rule_fsm->GetFsm().IsScanableState(i)) { continue; } add_task_adaptive_token_mask(cur_stack_element, rule_id == root_rule_id); } } if (max_threads_ > 1) { thread_pool->Join(); } return CompiledGrammar(compiled_grammar_impl); } CompiledGrammar GrammarCompilerSub::CompileBuiltinJSONGrammar() { return MultiThreadCompileGrammar(Grammar::BuiltinJSONGrammar()); } CompiledGrammar GrammarCompilerSub::CompileJSONSchema( const std::string& schema, bool any_whitespace, std::optional indent, std::optional> separators, bool strict_mode, std::optional max_whitespace_cnt, bool any_order ) { return MultiThreadCompileGrammar(Grammar::FromJSONSchema( schema, any_whitespace, indent, separators, strict_mode, max_whitespace_cnt, /*print_converted_ebnf=*/false, any_order )); } CompiledGrammar GrammarCompilerSub::CompileStructuralTag(const std::string& structural_tag_json) { auto result = Grammar::FromStructuralTag(structural_tag_json, tokenizer_info_); XGRAMMAR_CHECK(std::holds_alternative(result)) << GetMessageFromVariantError(std::get<1>(result)); return MultiThreadCompileGrammar(std::get<0>(result)); } CompiledGrammar GrammarCompilerSub::CompileRegex(const std::string& regex) { return MultiThreadCompileGrammar(Grammar::FromRegex(regex)); } CompiledGrammar GrammarCompilerSub::CompileGrammar(const Grammar& grammar) { return MultiThreadCompileGrammar(grammar); } CompiledGrammar GrammarCompilerSub::CompileGrammar( const std::string& ebnf_str, std::string root_rule_name ) { return MultiThreadCompileGrammar(Grammar::FromEBNF(ebnf_str, root_rule_name)); } void GrammarCompilerSub::TagDispatchOptimization( std::shared_ptr compiled_grammar_impl, std::unordered_map* tag_dispatch_rule_id_to_second_slicing_bitset ) { using GrammarExprType = Grammar::Impl::GrammarExprType; tag_dispatch_rule_id_to_second_slicing_bitset->clear(); // Optimization for TagDispatch: Precompute the definitely accepted tokens. for (int i = 0; i < compiled_grammar_impl->grammar->NumRules(); i++) { const auto& rule = compiled_grammar_impl->grammar->GetRule(i); const auto& rule_body = compiled_grammar_impl->grammar->GetGrammarExpr(rule.body_expr_id); if (rule_body.type != GrammarExprType::kTagDispatch) { continue; } XGRAMMAR_DCHECK(rule_body.type == GrammarExprType::kTagDispatch); Grammar::Impl::TagDispatch tag_dispatch = compiled_grammar_impl->GetGrammar()->GetTagDispatch(rule.body_expr_id); const auto& sorted_decoded_vocab = tokenizer_info_.GetSortedDecodedVocab(); DynamicBitset definite_accepted_tokens_since_second_char(sorted_decoded_vocab.size()); for (int j = 0; j < static_cast(sorted_decoded_vocab.size()); j++) { bool definite_accept_since_second_char = true; const auto& token = sorted_decoded_vocab[j].second; if (token.empty()) { definite_accepted_tokens_since_second_char.Set(j); continue; } // Check if the token contains any string trigger or exclude string after first char. for (const auto& [trigger, rule_id] : tag_dispatch.tag_rule_pairs) { if (token.find(trigger, 1) != std::string::npos) { definite_accept_since_second_char = false; break; } } if (definite_accept_since_second_char) { for (const auto& excl : tag_dispatch.excludes) { if (token.find(excl, 1) != std::string::npos) { definite_accept_since_second_char = false; break; } } } if (definite_accept_since_second_char) { definite_accepted_tokens_since_second_char.Set(j); } } (*tag_dispatch_rule_id_to_second_slicing_bitset)[i] = definite_accepted_tokens_since_second_char; } } /******************* GrammarCompiler::Impl *******************/ /*! * \brief The keys for the cache. This is defined here instead of inside the GrammarCompiler::Impl * class due C++ template specialization and hash specialization rules. */ class GrammarCompilerCacheKeys { public: struct SchemaKey { std::string schema; bool any_whitespace; std::optional indent; std::optional> separators; bool strict_mode; std::optional max_whitespace_cnt; bool any_order; XGRAMMAR_EQUAL_BY_MEMBERS( SchemaKey, &SchemaKey::schema, &SchemaKey::any_whitespace, &SchemaKey::indent, &SchemaKey::separators, &SchemaKey::strict_mode, &SchemaKey::max_whitespace_cnt, &SchemaKey::any_order ); }; struct StructuralTagKey { std::string structural_tag_json; XGRAMMAR_EQUAL_BY_MEMBERS(StructuralTagKey, &StructuralTagKey::structural_tag_json); }; struct GrammarKey { std::string ebnf_str; std::string root_rule_name; XGRAMMAR_EQUAL_BY_MEMBERS(GrammarKey, &GrammarKey::ebnf_str, &GrammarKey::root_rule_name); }; struct RegexKey { std::string regex; XGRAMMAR_EQUAL_BY_MEMBERS(RegexKey, &RegexKey::regex); }; struct BuiltinJSONGrammarKey { XGRAMMAR_EQUAL_BY_MEMBERS_EMPTY(BuiltinJSONGrammarKey); }; using UnionKey = std::variant; }; } // namespace xgrammar XGRAMMAR_HASH_BY_MEMBERS( xgrammar::GrammarCompilerCacheKeys::SchemaKey, &xgrammar::GrammarCompilerCacheKeys::SchemaKey::schema, &xgrammar::GrammarCompilerCacheKeys::SchemaKey::any_whitespace, &xgrammar::GrammarCompilerCacheKeys::SchemaKey::indent, &xgrammar::GrammarCompilerCacheKeys::SchemaKey::separators, &xgrammar::GrammarCompilerCacheKeys::SchemaKey::strict_mode, &xgrammar::GrammarCompilerCacheKeys::SchemaKey::max_whitespace_cnt, &xgrammar::GrammarCompilerCacheKeys::SchemaKey::any_order ); XGRAMMAR_HASH_BY_MEMBERS( xgrammar::GrammarCompilerCacheKeys::StructuralTagKey, &xgrammar::GrammarCompilerCacheKeys::StructuralTagKey::structural_tag_json ); XGRAMMAR_HASH_BY_MEMBERS( xgrammar::GrammarCompilerCacheKeys::GrammarKey, &xgrammar::GrammarCompilerCacheKeys::GrammarKey::ebnf_str, &xgrammar::GrammarCompilerCacheKeys::GrammarKey::root_rule_name ); XGRAMMAR_HASH_BY_MEMBERS( xgrammar::GrammarCompilerCacheKeys::RegexKey, &xgrammar::GrammarCompilerCacheKeys::RegexKey::regex ); XGRAMMAR_HASH_BY_MEMBERS_EMPTY(xgrammar::GrammarCompilerCacheKeys::BuiltinJSONGrammarKey); namespace xgrammar { /*! * \brief The implementation of the grammar compiler with cache. It calls the no cache compiler * to compile the grammar, and implements the cache logic upon it. */ class GrammarCompiler::Impl { public: Impl( const TokenizerInfo& tokenizer_info, int max_threads, bool cache_enabled, int64_t max_memory_bytes ) : cache_enabled_(cache_enabled), rule_level_cache_( cache_enabled ? std::optional( max_memory_bytes == -1 ? static_cast(-1) : static_cast(max_memory_bytes - max_memory_bytes / 3 * 2) ) : std::nullopt ), no_cache_compiler_(tokenizer_info, max_threads, rule_level_cache_), grammar_level_cache_( max_memory_bytes == -1 ? static_cast(-1) : static_cast(max_memory_bytes / 3 * 2), Computer(*this) ) { if (max_memory_bytes < -1) { XGRAMMAR_LOG(FATAL) << "Invalid max_memory_bytes: " << max_memory_bytes << ". " << "It should be -1 (unlimited) or a non-negative integer."; } } CompiledGrammar CompileBuiltinJSONGrammar(); CompiledGrammar CompileJSONSchema( const std::string& schema, bool any_whitespace, std::optional indent, std::optional> separators, bool strict_mode, std::optional max_whitespace_cnt, bool any_order ); CompiledGrammar CompileStructuralTag(const std::string& structural_tag_json); CompiledGrammar CompileRegex(const std::string& regex); CompiledGrammar CompileGrammar(const Grammar& grammar); CompiledGrammar CompileGrammar(const std::string& ebnf_str, std::string root_rule_name); void ClearCache(); int64_t GetCacheSizeBytes() const; int64_t CacheLimitBytes() const; private: using SchemaKey = GrammarCompilerCacheKeys::SchemaKey; using StructuralTagKey = GrammarCompilerCacheKeys::StructuralTagKey; using GrammarKey = GrammarCompilerCacheKeys::GrammarKey; using RegexKey = GrammarCompilerCacheKeys::RegexKey; using BuiltinJSONGrammarKey = GrammarCompilerCacheKeys::BuiltinJSONGrammarKey; using UnionKey = GrammarCompilerCacheKeys::UnionKey; CompiledGrammar Compute(const UnionKey& key); struct Computer { Computer(Impl& compiler) : compiler(compiler) {} // Forward the key to GrammarCompiler::Impl::Compute(key) CompiledGrammar operator()(const UnionKey& key) const { return compiler.Compute(key); } GrammarCompiler::Impl& compiler; }; struct SizeEstimator { std::size_t operator()(const CompiledGrammar& value) const { return value.MemorySizeBytes(); } }; /*! \brief Whether the cache is enabled. */ const bool cache_enabled_; /*! \brief The crossing cache manager for compiled grammars. */ std::optional rule_level_cache_ = std::nullopt; /*! \brief The no cache compiler. */ GrammarCompilerSub no_cache_compiler_; /*! \brief The cache for compiled grammars. */ ThreadSafeLRUCache grammar_level_cache_; }; CompiledGrammar GrammarCompiler::Impl::Compute(const UnionKey& key) { return std::visit( [this](const auto& key) -> CompiledGrammar { using KeyType = std::decay_t; if constexpr (std::is_same_v) { const auto& [ebnf_str, root_rule_name] = key; return this->no_cache_compiler_.CompileGrammar(ebnf_str, root_rule_name); } else if constexpr (std::is_same_v) { const auto& [schema, any_whitespace, indent, separators, strict_mode, max_whitespace_cnt, any_order] = key; return this->no_cache_compiler_.CompileJSONSchema( schema, any_whitespace, indent, separators, strict_mode, max_whitespace_cnt, any_order ); } else if constexpr (std::is_same_v) { const auto& [structural_tag_json] = key; return this->no_cache_compiler_.CompileStructuralTag(structural_tag_json); } else if constexpr (std::is_same_v) { const auto& [regex] = key; return this->no_cache_compiler_.CompileRegex(regex); } else if constexpr (std::is_same_v) { return this->no_cache_compiler_.CompileBuiltinJSONGrammar(); } else { XGRAMMAR_UNREACHABLE(); } }, key ); } CompiledGrammar GrammarCompiler::Impl::CompileBuiltinJSONGrammar() { if (!cache_enabled_) { return no_cache_compiler_.CompileBuiltinJSONGrammar(); } return grammar_level_cache_.Get(BuiltinJSONGrammarKey{}); } CompiledGrammar GrammarCompiler::Impl::CompileJSONSchema( const std::string& schema, bool any_whitespace, std::optional indent, std::optional> separators, bool strict_mode, std::optional max_whitespace_cnt, bool any_order ) { if (!cache_enabled_) { return no_cache_compiler_.CompileJSONSchema( schema, any_whitespace, indent, separators, strict_mode, max_whitespace_cnt, any_order ); } return grammar_level_cache_.Get(SchemaKey{ schema, any_whitespace, indent, separators, strict_mode, max_whitespace_cnt, any_order }); } CompiledGrammar GrammarCompiler::Impl::CompileStructuralTag(const std::string& structural_tag_json ) { if (!cache_enabled_) { return no_cache_compiler_.CompileStructuralTag(structural_tag_json); } return grammar_level_cache_.Get(StructuralTagKey{structural_tag_json}); } CompiledGrammar GrammarCompiler::Impl::CompileRegex(const std::string& regex) { if (!cache_enabled_) { return no_cache_compiler_.CompileRegex(regex); } return grammar_level_cache_.Get(RegexKey{regex}); } CompiledGrammar GrammarCompiler::Impl::CompileGrammar(const Grammar& grammar) { if (!cache_enabled_) { return no_cache_compiler_.CompileGrammar(grammar); } return grammar_level_cache_.Get(GrammarKey{grammar.ToString(), grammar->GetRootRule().name}); } CompiledGrammar GrammarCompiler::Impl::CompileGrammar( const std::string& ebnf_str, std::string root_rule_name ) { if (!cache_enabled_) { return no_cache_compiler_.CompileGrammar(ebnf_str, root_rule_name); } return grammar_level_cache_.Get(GrammarKey{ebnf_str, root_rule_name}); } void GrammarCompiler::Impl::ClearCache() { grammar_level_cache_.Clear(); if (rule_level_cache_.has_value()) { rule_level_cache_->ClearCache(); } } int64_t GrammarCompiler::Impl::GetCacheSizeBytes() const { return static_cast(grammar_level_cache_.MemorySize()) + static_cast(MemorySize(rule_level_cache_)); } int64_t GrammarCompiler::Impl::CacheLimitBytes() const { const auto size = grammar_level_cache_.MaxMemorySize(); if (size == grammar_level_cache_.kUnlimitedSize) return -1; return static_cast(size) + (rule_level_cache_.has_value() ? static_cast(rule_level_cache_->GetMaxSize()) : 0); } /******************* GrammarCompiler *******************/ GrammarCompiler::GrammarCompiler( const TokenizerInfo& tokenizer_info, int max_threads, bool cache_enabled, int64_t max_memory_bytes ) : pimpl_(std::make_shared(tokenizer_info, max_threads, cache_enabled, max_memory_bytes)) { } CompiledGrammar GrammarCompiler::CompileJSONSchema( const std::string& schema, bool any_whitespace, std::optional indent, std::optional> separators, bool strict_mode, std::optional max_whitespace_cnt, bool any_order ) { return pimpl_->CompileJSONSchema( schema, any_whitespace, indent, separators, strict_mode, max_whitespace_cnt, any_order ); } CompiledGrammar GrammarCompiler::CompileBuiltinJSONGrammar() { return pimpl_->CompileBuiltinJSONGrammar(); } CompiledGrammar GrammarCompiler::CompileStructuralTag(const std::string& structural_tag_json) { return pimpl_->CompileStructuralTag(structural_tag_json); } CompiledGrammar GrammarCompiler::CompileRegex(const std::string& regex) { return pimpl_->CompileRegex(regex); } CompiledGrammar GrammarCompiler::CompileGrammar(const Grammar& grammar) { return pimpl_->CompileGrammar(grammar); } CompiledGrammar GrammarCompiler::CompileGrammar( const std::string& ebnf_str, const std::string& root_rule_name ) { return pimpl_->CompileGrammar(ebnf_str, root_rule_name); } void GrammarCompiler::ClearCache() { pimpl_->ClearCache(); } int64_t GrammarCompiler::GetCacheSizeBytes() const { return pimpl_->GetCacheSizeBytes(); } int64_t GrammarCompiler::CacheLimitBytes() const { return pimpl_->CacheLimitBytes(); } } // namespace xgrammar xgrammar-0.2.3/cpp/grammar_functor.cc000066400000000000000000003113061521764210300176160ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar_functor.cc */ #include "grammar_functor.h" #include #include #include #include #include #include #include #include #include #include #include #include #include "compiled_grammar_impl.h" #include "fsm.h" #include "fsm_builder.h" #include "grammar_builder.h" #include "grammar_impl.h" #include "support/container.h" #include "support/encoding.h" #include "support/logging.h" #include "xgrammar/grammar.h" namespace xgrammar { using GrammarExpr = Grammar::Impl::GrammarExpr; using ExprType = Grammar::Impl::GrammarExprType; /*************************** Impl of grammar constructors ***************************/ /*! * \brief Base class for grammar mutators that add subgrammars. * * Provides functionality to visit a subgrammar and add its rules to the builder * while maintaining proper rule references and names. */ class SubGrammarAdderImpl : public GrammarMutator { public: SubGrammarAdderImpl() = default; /*! * \brief Visit a subgrammar and add the rules to the builder. * \param grammar The subgrammar to visit. * \return The new id of the root rule of this subgrammar. */ int32_t ApplyWithBuilder(GrammarBuilder* builder, const Grammar& sub_grammar) { InitGrammar(sub_grammar); InitBuilder(builder); new_rule_ids_names.reserve(base_grammar_->NumRules()); new_rule_ids_names.clear(); for (int i = 0; i < static_cast(base_grammar_->NumRules()); ++i) { auto new_name = builder_->GetNewRuleName(base_grammar_->GetRule(i).name); auto new_id = builder_->AddEmptyRule(new_name); new_rule_ids_names.emplace_back(new_id, new_name); } for (int i = 0; i < static_cast(base_grammar_->NumRules()); ++i) { auto rule = base_grammar_->GetRule(i); cur_rule_name_ = new_rule_ids_names[i].second; auto new_body_expr_id = VisitExpr(rule.body_expr_id); builder_->UpdateRuleBody(new_rule_ids_names[i].first, new_body_expr_id); auto new_lookahead_assertion_id = VisitLookaheadAssertion(rule.lookahead_assertion_id); builder_->UpdateLookaheadAssertion(new_rule_ids_names[i].first, new_lookahead_assertion_id); } return new_rule_ids_names[base_grammar_->GetRootRuleId()].first; } int32_t VisitRuleRef(const GrammarExpr& grammar_expr) final { return builder_->AddRuleRef(new_rule_ids_names[grammar_expr[0]].first); } int32_t VisitRepeat(const GrammarExpr& grammar_expr) final { return builder_->AddRepeat( new_rule_ids_names[grammar_expr[0]].first, grammar_expr[1], grammar_expr[2] ); } int32_t VisitTagDispatch(const GrammarExpr& grammar_expr) final { Grammar::Impl::TagDispatch old_tag_dispatch = base_grammar_->GetTagDispatch(grammar_expr); Grammar::Impl::TagDispatch new_tag_dispatch; for (const auto& [trigger, rule_id] : old_tag_dispatch.tag_rule_pairs) { new_tag_dispatch.tag_rule_pairs.emplace_back(trigger, new_rule_ids_names[rule_id].first); } new_tag_dispatch.loop_after_dispatch = old_tag_dispatch.loop_after_dispatch; new_tag_dispatch.excludes = old_tag_dispatch.excludes; return builder_->AddTagDispatch(new_tag_dispatch); } int32_t VisitTokenTagDispatch(const GrammarExpr& grammar_expr) final { Grammar::Impl::TokenTagDispatch old_ttd = base_grammar_->GetTokenTagDispatch(grammar_expr); Grammar::Impl::TokenTagDispatch new_ttd; for (const auto& [token_id, rule_id] : old_ttd.trigger_rule_pairs) { new_ttd.trigger_rule_pairs.emplace_back(token_id, new_rule_ids_names[rule_id].first); } new_ttd.loop_after_dispatch = old_ttd.loop_after_dispatch; new_ttd.excludes = old_ttd.excludes; return builder_->AddTokenTagDispatch(new_ttd); } std::vector> new_rule_ids_names; }; /*! * \brief Implementation of grammar union operation. * * Creates a new grammar that accepts strings from any of the input grammars. * The resulting grammar has a new root rule that chooses between the root rules * of all input grammars. */ class GrammarUnionFunctorImpl : public GrammarMutator { public: GrammarUnionFunctorImpl() = default; Grammar Apply(const std::vector& grammars) { InitGrammar(); InitBuilder(); auto root_rule_id = builder_->AddEmptyRule("root"); std::vector new_root_choices; new_root_choices.reserve(grammars.size()); for (const auto& grammar : grammars) { auto new_root_id_for_grammar = SubGrammarAdderImpl().ApplyWithBuilder(builder_, grammar); auto new_rule_ref = builder_->AddRuleRef(new_root_id_for_grammar); auto new_rule_ref_seq = builder_->AddSequence({new_rule_ref}); new_root_choices.push_back(new_rule_ref_seq); } builder_->UpdateRuleBody(root_rule_id, builder_->AddChoices(new_root_choices)); return builder_->Get(root_rule_id); } // Avoid hiding the original Apply(const Grammar&) Grammar Apply(const Grammar& grammar) final { XGRAMMAR_LOG(FATAL) << "Should not be called"; XGRAMMAR_UNREACHABLE(); } }; /*! * \brief Implementation of grammar concatenation operation. * * Creates a new grammar that accepts strings that are concatenations of strings * from the input grammars in order. The resulting grammar has a new root rule * that concatenates the root rules of all input grammars. */ class GrammarConcatFunctorImpl : public GrammarMutator { public: GrammarConcatFunctorImpl() = default; Grammar Apply(const std::vector& grammars) { InitGrammar(); InitBuilder(); auto root_rule_id = builder_->AddEmptyRule("root"); std::vector new_root_sequence; new_root_sequence.reserve(grammars.size()); for (const auto& grammar : grammars) { auto new_root_id_for_grammar = SubGrammarAdderImpl().ApplyWithBuilder(builder_, grammar); auto new_rule_ref = builder_->AddRuleRef(new_root_id_for_grammar); new_root_sequence.push_back(new_rule_ref); } auto new_root_seq = builder_->AddSequence(new_root_sequence); builder_->UpdateRuleBody(root_rule_id, builder_->AddChoices({new_root_seq})); return builder_->Get(root_rule_id); } // Avoid hiding the original Apply(const Grammar&) Grammar Apply(const Grammar& grammar) final { XGRAMMAR_LOG(FATAL) << "Should not be called"; XGRAMMAR_UNREACHABLE(); } }; /*************************** Impl of grammar normalizers ***************************/ /*! * \brief Eliminates single-element sequence or choice or character class in the grammar. * \example `A ::= choices("a")` --> `A ::= "a"` (the body is a string) * \example `A ::= sequence("a")` --> `A ::= "a"` (the body is a string) * \example `A ::= [a-a]` --> `A ::= "a"` (the body is a string) */ class SingleElementExprEliminator : public GrammarMutator { public: using GrammarMutator::Apply; using GrammarMutator::GrammarMutator; private: int32_t VisitSequence(const GrammarExpr& grammar_expr) final { std::vector sequence_ids; for (int32_t i : grammar_expr) { sequence_ids.push_back(VisitExpr(i)); } if (sequence_ids.size() == 1) { return sequence_ids[0]; } return builder_->AddSequence(sequence_ids); } int32_t VisitChoices(const GrammarExpr& grammar_expr) final { std::vector choice_ids; for (int32_t i : grammar_expr) { choice_ids.push_back(VisitExpr(i)); } if (choice_ids.size() == 1) { return choice_ids[0]; } return builder_->AddChoices(choice_ids); } int32_t VisitCharacterClass(const GrammarExpr& grammar_expr) final { if (grammar_expr.data_len == 3 && grammar_expr[0] == 0 && grammar_expr[1] == grammar_expr[2]) { std::string str = CharToUTF8(grammar_expr[1]); std::vector bytes; bytes.reserve(str.size()); for (char c : str) { bytes.push_back(static_cast(c)); } return builder_->AddByteString(bytes); } return builder_->AddGrammarExpr(grammar_expr); } }; /*! * \brief Take a grammar from SingleElementExprEliminator and normalize the structure of the * grammar. * * \note The normalized form: * Each rule should be either: * - A sequence of choices, each choice is a sequence of elements. Elements can be a character * class, a byte string, or a rule reference. Only the first choice can be an empty string, * indicating the rule can be empty. E.g. * `rule_name ::= ("" | (element1_1 element1_2 ...) | (element2_1 element2_2 ...) | ...)` * - A macro. Now only TagDispatch is supported. * * The lookahead assertion should be a sequence. * * New rules may be created to make every rule fit the normalized form. * * \example `A ::= ((a) (((b)) (c)) "")` -> `A ::= ((a b c))` * \example `A ::= (a | (b | (c | "")))` -> `A ::= ("" | (a) | (b) | (c))` * \example `A ::= (a | (b (c | d)))` -> `A ::= ((a) | (b A_1)), A_1 ::= ((c) | (d))` * \example `A ::= (a | TagDispatch((tag1, rule1)))` -> `A ::= ((a) | (A_1)), A_1 ::= * TagDispatch((tag1, rule1))` */ class StructureNormalizerImpl : public GrammarMutator { public: using GrammarMutator::GrammarMutator; Grammar Apply(const Grammar& grammar) final { auto grammar_new = SingleElementExprEliminator().Apply(grammar); InitGrammar(grammar_new); InitBuilder(); for (int i = 0; i < static_cast(base_grammar_->NumRules()); ++i) { builder_->AddEmptyRule(base_grammar_->GetRule(i).name); } for (int i = 0; i < static_cast(base_grammar_->NumRules()); ++i) { auto rule = base_grammar_->GetRule(i); auto grammar_expr = base_grammar_->GetGrammarExpr(rule.body_expr_id); cur_rule_name_ = rule.name; auto new_body_expr_id = VisitRuleBody(grammar_expr); builder_->UpdateRuleBody(i, new_body_expr_id); builder_->UpdateLookaheadAssertion(i, VisitLookaheadAssertion(rule.lookahead_assertion_id)); } return builder_->Get(base_grammar_->GetRootRule().name); } private: int32_t VisitLookaheadAssertion(int32_t lookahead_assertion_id) final { if (lookahead_assertion_id == -1) { return -1; } auto assertion_expr = base_grammar_->GetGrammarExpr(lookahead_assertion_id); switch (assertion_expr.type) { case GrammarExprType::kSequence: return builder_->AddSequence(VisitSequence_(assertion_expr)); case GrammarExprType::kChoices: XGRAMMAR_LOG(FATAL) << "Choices in lookahead assertion are not supported yet"; XGRAMMAR_UNREACHABLE(); case GrammarExprType::kEmptyStr: XGRAMMAR_LOG(FATAL) << "Empty string should not be in lookahead assertion"; XGRAMMAR_UNREACHABLE(); case GrammarExprType::kTagDispatch: XGRAMMAR_LOG(FATAL) << "TagDispatch should not be in lookahead assertion"; XGRAMMAR_UNREACHABLE(); case GrammarExprType::kByteString: case GrammarExprType::kCharacterClass: case GrammarExprType::kCharacterClassStar: case GrammarExprType::kRuleRef: case GrammarExprType::kRepeat: case GrammarExprType::kToken: case GrammarExprType::kExcludeToken: case GrammarExprType::kTokenTagDispatch: return builder_->AddSequence({builder_->AddGrammarExpr(assertion_expr)}); default: XGRAMMAR_LOG(FATAL) << "Unexpected lookahead assertion type: " << static_cast(assertion_expr.type); XGRAMMAR_UNREACHABLE(); } } /*! \brief Visit a GrammarExpr as a rule body. */ int32_t VisitRuleBody(const GrammarExpr& grammar_expr) { switch (grammar_expr.type) { case GrammarExprType::kSequence: return builder_->AddChoices({builder_->AddSequence(VisitSequence_(grammar_expr))}); case GrammarExprType::kChoices: return builder_->AddChoices(VisitChoices_(grammar_expr)); case GrammarExprType::kEmptyStr: return builder_->AddChoices({builder_->AddEmptyStr()}); case GrammarExprType::kByteString: case GrammarExprType::kCharacterClass: case GrammarExprType::kCharacterClassStar: case GrammarExprType::kRuleRef: case GrammarExprType::kRepeat: case GrammarExprType::kToken: case GrammarExprType::kExcludeToken: return builder_->AddChoices({builder_->AddSequence({builder_->AddGrammarExpr(grammar_expr)}) }); case GrammarExprType::kTagDispatch: return VisitTagDispatch(grammar_expr); case GrammarExprType::kTokenTagDispatch: { auto ttd_expr_id = VisitTokenTagDispatch(grammar_expr); auto new_rule_id = builder_->AddRuleWithHint(cur_rule_name_, ttd_expr_id); return builder_->AddChoices({builder_->AddSequence({builder_->AddRuleRef(new_rule_id)})}); } default: XGRAMMAR_LOG(FATAL) << "Unexpected sequence type: " << static_cast(grammar_expr.type); XGRAMMAR_UNREACHABLE(); } } /*! * \brief Visit a GrammarExpr containing choices. * \returns A list of new choice GrammarExpr ids. */ std::vector VisitChoices_(const GrammarExpr& grammar_expr) { std::vector new_choice_ids; bool found_empty = false; for (auto i : grammar_expr) { auto choice_expr = base_grammar_->GetGrammarExpr(i); switch (choice_expr.type) { case GrammarExprType::kSequence: VisitSequenceInChoices(choice_expr, &new_choice_ids, &found_empty); break; case GrammarExprType::kChoices: VisitChoicesInChoices(choice_expr, &new_choice_ids, &found_empty); break; case GrammarExprType::kEmptyStr: found_empty = true; break; case GrammarExprType::kByteString: case GrammarExprType::kCharacterClass: case GrammarExprType::kCharacterClassStar: case GrammarExprType::kRuleRef: case GrammarExprType::kRepeat: case GrammarExprType::kToken: case GrammarExprType::kExcludeToken: VisitElementInChoices(choice_expr, &new_choice_ids); break; case GrammarExprType::kTagDispatch: { auto tag_dispatch_expr_id = VisitTagDispatch(choice_expr); auto new_rule_id = builder_->AddRuleWithHint(cur_rule_name_, tag_dispatch_expr_id); auto new_sequence_id = builder_->AddSequence({builder_->AddRuleRef(new_rule_id)}); new_choice_ids.push_back(new_sequence_id); break; } case GrammarExprType::kTokenTagDispatch: { auto ttd_expr_id = VisitTokenTagDispatch(choice_expr); auto new_rule_id = builder_->AddRuleWithHint(cur_rule_name_, ttd_expr_id); auto new_sequence_id = builder_->AddSequence({builder_->AddRuleRef(new_rule_id)}); new_choice_ids.push_back(new_sequence_id); break; } default: XGRAMMAR_LOG(FATAL) << "Unexpected choice type: " << static_cast(choice_expr.type); } } if (found_empty) { new_choice_ids.insert(new_choice_ids.begin(), builder_->AddEmptyStr()); } XGRAMMAR_ICHECK(new_choice_ids.size() >= 1); return new_choice_ids; } /*! \brief Visit a sequence GrammarExpr that is one of a list of choices. */ void VisitSequenceInChoices( const GrammarExpr& grammar_expr, std::vector* new_choice_ids, bool* found_empty ) { auto sub_sequence_ids = VisitSequence_(grammar_expr); if (sub_sequence_ids.size() == 0) { *found_empty = true; } else { new_choice_ids->push_back(builder_->AddSequence(sub_sequence_ids)); } } /*! \brief Visit a choice GrammarExpr that is one of a list of choices. */ void VisitChoicesInChoices( const GrammarExpr& grammar_expr, std::vector* new_choice_ids, bool* found_empty ) { auto sub_choice_ids = VisitChoices_(grammar_expr); bool contains_empty = builder_->GetGrammarExpr(sub_choice_ids[0]).type == GrammarExprType::kEmptyStr; if (contains_empty) { *found_empty = true; new_choice_ids->insert( new_choice_ids->end(), sub_choice_ids.begin() + 1, sub_choice_ids.end() ); } else { new_choice_ids->insert(new_choice_ids->end(), sub_choice_ids.begin(), sub_choice_ids.end()); } } /*! \brief Visit an atom element GrammarExpr that is one of a list of choices. */ void VisitElementInChoices( const GrammarExpr& grammar_expr, std::vector* new_choice_ids ) { auto sub_expr_id = builder_->AddGrammarExpr(grammar_expr); new_choice_ids->push_back(builder_->AddSequence({sub_expr_id})); } /*! * \brief Visit a GrammarExpr containing a sequence. * \returns A list of new sequence GrammarExpr ids. */ std::vector VisitSequence_(const GrammarExpr& grammar_expr) { std::vector new_sequence_ids; for (auto i : grammar_expr) { auto element_expr = base_grammar_->GetGrammarExpr(i); switch (element_expr.type) { case GrammarExprType::kSequence: VisitSequenceInSequence(element_expr, &new_sequence_ids); break; case GrammarExprType::kChoices: VisitChoiceInSequence(element_expr, &new_sequence_ids); break; case GrammarExprType::kEmptyStr: break; case GrammarExprType::kByteString: case GrammarExprType::kCharacterClass: case GrammarExprType::kCharacterClassStar: case GrammarExprType::kRuleRef: case GrammarExprType::kRepeat: case GrammarExprType::kToken: case GrammarExprType::kExcludeToken: VisitElementInSequence(element_expr, &new_sequence_ids); break; case GrammarExprType::kTagDispatch: { auto tag_dispatch_expr_id = VisitTagDispatch(element_expr); auto new_rule_id = builder_->AddRuleWithHint(cur_rule_name_, tag_dispatch_expr_id); new_sequence_ids.push_back(builder_->AddRuleRef(new_rule_id)); break; } case GrammarExprType::kTokenTagDispatch: { auto ttd_expr_id = VisitTokenTagDispatch(element_expr); auto new_rule_id = builder_->AddRuleWithHint(cur_rule_name_, ttd_expr_id); new_sequence_ids.push_back(builder_->AddRuleRef(new_rule_id)); break; } default: XGRAMMAR_LOG(FATAL) << "Unexpected sequence type: " << static_cast(element_expr.type); } } return new_sequence_ids; } /*! \brief Visit a sequence GrammarExpr that is one element in another sequence. */ void VisitSequenceInSequence( const GrammarExpr& grammar_expr, std::vector* new_sequence_ids ) { auto sub_sequence_ids = VisitSequence_(grammar_expr); new_sequence_ids->insert( new_sequence_ids->end(), sub_sequence_ids.begin(), sub_sequence_ids.end() ); } /*! \brief Visit a choice GrammarExpr that is one element in a sequence. */ void VisitChoiceInSequence( const GrammarExpr& grammar_expr, std::vector* new_sequence_ids ) { auto sub_choice_ids = VisitChoices_(grammar_expr); if (sub_choice_ids.size() == 1) { auto choice_element_expr = builder_->GetGrammarExpr(sub_choice_ids[0]); if (choice_element_expr.type != GrammarExprType::kEmptyStr) { new_sequence_ids->insert( new_sequence_ids->end(), choice_element_expr.begin(), choice_element_expr.end() ); } } else { auto new_choice_id = builder_->AddChoices(sub_choice_ids); auto new_choice_rule_id = builder_->AddRuleWithHint(cur_rule_name_, new_choice_id); new_sequence_ids->push_back(builder_->AddRuleRef(new_choice_rule_id)); } } /*! \brief Visit an atom element GrammarExpr that is in a sequence. */ void VisitElementInSequence( const GrammarExpr& grammar_expr, std::vector* new_sequence_ids ) { new_sequence_ids->push_back(builder_->AddGrammarExpr(grammar_expr)); } }; /*! * \brief A class that normalizes a grammar by applying a series of transformations. * * The normalizer applies the following transformations in order: * 1. SingleElementExprEliminator - Eliminates single element expressions * 2. NestedRuleUnwrapper - Unwraps nested rules */ class GrammarNormalizerImpl { public: GrammarNormalizerImpl() = default; Grammar Apply(const Grammar& grammar) { auto renamed_grammar = RootRuleRenamer::Apply(grammar); return StructureNormalizerImpl().Apply(renamed_grammar); } }; /*************************** Impl of grammar optimizers ***************************/ /*! * \brief Inline rules that can be inlined. * * Now we only inline rule references that: * 1. at the beginning of a sequence * 2. The rule should be a sequence of choices, cannot be empty, cannot refer to other rules */ class RuleInlinerImpl : public GrammarMutator { public: using GrammarMutator::Apply; using GrammarMutator::GrammarMutator; private: int32_t VisitChoices(const GrammarExpr& grammar_expr) final { std::vector new_choice_ids; for (int i : grammar_expr) { auto choice_expr = base_grammar_->GetGrammarExpr(i); if (choice_expr.type == GrammarExprType::kEmptyStr) { new_choice_ids.push_back(VisitExpr(i)); continue; } XGRAMMAR_ICHECK(choice_expr.type == GrammarExprType::kSequence); auto first_element = base_grammar_->GetGrammarExpr(choice_expr[0]); if (first_element.type != GrammarExprType::kRuleRef) { new_choice_ids.push_back(VisitExpr(choice_expr)); continue; } auto rule_ref_id = first_element[0]; if (can_rule_be_inlined_.count(rule_ref_id) == 0) { can_rule_be_inlined_[rule_ref_id] = CheckIfRuleCanBeInlined(rule_ref_id); } if (!can_rule_be_inlined_[rule_ref_id]) { new_choice_ids.push_back(VisitExpr(choice_expr)); continue; } // Do inlining std::vector other_elements; for (int i = 1; i < choice_expr.size(); ++i) { other_elements.push_back(VisitExpr(choice_expr[i])); } auto ref_rule = base_grammar_->GetRule(rule_ref_id); auto ref_grammar_expr = base_grammar_->GetGrammarExpr(ref_rule.body_expr_id); for (auto ref_choice_id : ref_grammar_expr) { auto ref_choice_expr = base_grammar_->GetGrammarExpr(ref_choice_id); XGRAMMAR_ICHECK(ref_choice_expr.type == GrammarExprType::kSequence); std::vector choice_to_add; for (auto ref_element_id : ref_choice_expr) { choice_to_add.push_back(VisitExpr(ref_element_id)); } choice_to_add.insert(choice_to_add.end(), other_elements.begin(), other_elements.end()); new_choice_ids.push_back(builder_->AddSequence(choice_to_add)); } } return builder_->AddChoices(new_choice_ids); } /** * The rule should be: a sequence of choices, cannot be empty, cannot refer to other rules */ bool CheckIfRuleCanBeInlined(int32_t rule_id) { auto rule = base_grammar_->GetRule(rule_id); auto grammar_expr = base_grammar_->GetGrammarExpr(rule.body_expr_id); if (grammar_expr.type != GrammarExprType::kChoices) { return false; } if (grammar_expr.size() == 0) { return false; } for (auto choice_id : grammar_expr) { auto choice_expr = base_grammar_->GetGrammarExpr(choice_id); if (choice_expr.type == GrammarExprType::kEmptyStr) { return false; } XGRAMMAR_ICHECK(choice_expr.type == GrammarExprType::kSequence); for (auto element_id : choice_expr) { auto element_expr = base_grammar_->GetGrammarExpr(element_id); if (element_expr.type == GrammarExprType::kRuleRef) { return false; } } } return true; } std::unordered_map can_rule_be_inlined_; }; /*! * \brief Analyze all referenced rules or the main rule. Return a list of all referenced rule ids. * This is useful for dead code elimination. */ class UsedRulesAnalyzer : public GrammarVisitor> { public: UsedRulesAnalyzer() = default; std::vector Apply(const Grammar& grammar) final { InitGrammar(grammar); std::set visited; std::queue().swap(visit_queue_); visit_queue_.push(base_grammar_->GetRootRuleId()); while (!visit_queue_.empty()) { auto rule_id = visit_queue_.front(); visit_queue_.pop(); if (visited.count(rule_id)) { continue; } visited.insert(rule_id); auto rule = base_grammar_->GetRule(rule_id); VisitExpr(rule.body_expr_id); if (rule.lookahead_assertion_id != -1) { VisitExpr(rule.lookahead_assertion_id); } } return std::vector(visited.begin(), visited.end()); } void VisitTagDispatch(const GrammarExpr& grammar_expr) { auto tag_dispatch = base_grammar_->GetTagDispatch(grammar_expr); for (const auto& [trigger, rule_id] : tag_dispatch.tag_rule_pairs) { visit_queue_.push(rule_id); } } void VisitTokenTagDispatch(const GrammarExpr& grammar_expr) { auto ttd = base_grammar_->GetTokenTagDispatch(grammar_expr); for (const auto& [token_id, rule_id] : ttd.trigger_rule_pairs) { visit_queue_.push(rule_id); } } void VisitRuleRef(const GrammarExpr& grammar_expr) { visit_queue_.push(grammar_expr[0]); } void VisitRepeat(const GrammarExpr& grammar_expr) { visit_queue_.push(grammar_expr[0]); } private: std::queue visit_queue_; }; class DeadCodeEliminatorImpl : public GrammarMutator { public: using GrammarMutator::Apply; using GrammarMutator::GrammarMutator; Grammar Apply(const Grammar& grammar) final { InitGrammar(grammar); InitBuilder(); auto used_rules = UsedRulesAnalyzer().Apply(grammar); rule_id_map_.clear(); for (auto rule_id : used_rules) { rule_id_map_[rule_id] = builder_->AddEmptyRule(grammar->GetRule(rule_id).name); } for (auto rule_id : used_rules) { auto rule = grammar->GetRule(rule_id); auto new_body_expr_id = VisitExpr(rule.body_expr_id); builder_->UpdateRuleBody(rule_id_map_[rule_id], new_body_expr_id); builder_->UpdateLookaheadAssertion( rule_id_map_[rule_id], VisitLookaheadAssertion(rule.lookahead_assertion_id) ); } XGRAMMAR_CHECK(rule_id_map_.count(grammar->GetRootRuleId()) > 0); return builder_->Get(rule_id_map_[grammar->GetRootRuleId()]); } int32_t VisitTagDispatch(const GrammarExpr& grammar_expr) final { Grammar::Impl::TagDispatch tag_dispatch = base_grammar_->GetTagDispatch(grammar_expr); for (auto& [trigger, rule_id] : tag_dispatch.tag_rule_pairs) { XGRAMMAR_DCHECK(rule_id_map_.count(rule_id) > 0); rule_id = rule_id_map_[rule_id]; } return builder_->AddTagDispatch(tag_dispatch); } int32_t VisitTokenTagDispatch(const GrammarExpr& grammar_expr) final { Grammar::Impl::TokenTagDispatch ttd = base_grammar_->GetTokenTagDispatch(grammar_expr); for (auto& [token_id, rule_id] : ttd.trigger_rule_pairs) { XGRAMMAR_DCHECK(rule_id_map_.count(rule_id) > 0); rule_id = rule_id_map_[rule_id]; } return builder_->AddTokenTagDispatch(ttd); } int32_t VisitRuleRef(const GrammarExpr& grammar_expr) final { XGRAMMAR_DCHECK(rule_id_map_.count(grammar_expr[0]) > 0); auto new_rule_id = rule_id_map_[grammar_expr[0]]; return builder_->AddRuleRef(new_rule_id); } int32_t VisitRepeat(const GrammarExpr& grammar_expr) final { XGRAMMAR_DCHECK(rule_id_map_.count(grammar_expr[0]) > 0); auto new_rule_id = rule_id_map_[grammar_expr[0]]; return builder_->AddRepeat(new_rule_id, grammar_expr[1], grammar_expr[2]); } private: std::unordered_map rule_id_map_; }; class LookaheadAssertionAnalyzerImpl : public GrammarMutator { public: using GrammarMutator::GrammarMutator; Grammar Apply(const Grammar& grammar) final { InitGrammar(grammar); InitBuilder(grammar); auto root_rule = grammar->GetRootRule(); auto root_grammar_expr = base_grammar_->GetGrammarExpr(root_rule.body_expr_id); if (root_grammar_expr.type == GrammarExprType::kTagDispatch || root_grammar_expr.type == GrammarExprType::kTokenTagDispatch) { return grammar; } BuildRuleLookaheadInfo(); for (int i = 0; i < static_cast(grammar->NumRules()); ++i) { auto rule = grammar->GetRule(i); if (i == grammar->GetRootRuleId()) { continue; } if (rule.lookahead_assertion_id != -1) { builder_->UpdateLookaheadExact(i, IsExactLookaheadAssertion(i)); continue; } auto look_head_assertion_id = DetectLookaheadAssertion(i); if (look_head_assertion_id != -1) { builder_->UpdateLookaheadAssertion(i, look_head_assertion_id); builder_->UpdateLookaheadExact(i); } } return builder_->Get(grammar->GetRootRuleId()); } bool IsExactLookaheadAssertion(int32_t rule_id) { XGRAMMAR_DCHECK(base_grammar_->GetRule(rule_id).lookahead_assertion_id != -1); return CanUseDerivedLookahead(rule_id); } int32_t DetectLookaheadAssertion(int32_t rule_id) { if (!CanUseDerivedLookahead(rule_id)) { return -1; } return builder_->AddSequence(rule_lookahead_infos_[rule_id].suffix_after_first_occurrence); } private: struct RuleLookaheadInfo { bool is_triggered_by_dispatch = false; bool appears_as_last_in_other_rule = false; int non_last_occurrence_count = 0; std::vector suffix_after_first_occurrence; }; bool CanUseDerivedLookahead(int32_t rule_id) const { const auto& info = rule_lookahead_infos_[rule_id]; return !info.is_triggered_by_dispatch && !info.appears_as_last_in_other_rule && info.non_last_occurrence_count == 1; } void BuildRuleLookaheadInfo() { rule_lookahead_infos_.assign(base_grammar_->NumRules(), RuleLookaheadInfo{}); for (int i = 0; i < static_cast(base_grammar_->NumRules()); ++i) { auto rule = base_grammar_->GetRule(i); auto grammar_expr = base_grammar_->GetGrammarExpr(rule.body_expr_id); if (grammar_expr.type == GrammarExprType::kTagDispatch) { auto tag_dispatch = base_grammar_->GetTagDispatch(grammar_expr); for (const auto& [trigger, rule_id] : tag_dispatch.tag_rule_pairs) { rule_lookahead_infos_[rule_id].is_triggered_by_dispatch = true; } continue; } if (grammar_expr.type == GrammarExprType::kTokenTagDispatch) { auto token_tag_dispatch = base_grammar_->GetTokenTagDispatch(grammar_expr); for (const auto& [token_id, rule_id] : token_tag_dispatch.trigger_rule_pairs) { rule_lookahead_infos_[rule_id].is_triggered_by_dispatch = true; } continue; } XGRAMMAR_DCHECK(grammar_expr.type == GrammarExprType::kChoices); for (auto sequence_id : grammar_expr) { auto sequence_expr = base_grammar_->GetGrammarExpr(sequence_id); if (sequence_expr.type != GrammarExprType::kSequence || sequence_expr.size() == 0) { continue; } auto last_element = base_grammar_->GetGrammarExpr(sequence_expr.end()[-1]); if (last_element.type == GrammarExprType::kRuleRef && i != last_element[0]) { rule_lookahead_infos_[last_element[0]].appears_as_last_in_other_rule = true; } for (int j = 0; j < sequence_expr.size() - 1; ++j) { auto element_expr = base_grammar_->GetGrammarExpr(sequence_expr[j]); if (element_expr.type != GrammarExprType::kRuleRef) { continue; } auto& info = rule_lookahead_infos_[element_expr[0]]; if (info.non_last_occurrence_count == 0) { info.suffix_after_first_occurrence.assign( sequence_expr.begin() + j + 1, sequence_expr.end() ); } ++info.non_last_occurrence_count; } } } } std::vector rule_lookahead_infos_; }; /*! * \brief Finds the rule reference graph of a grammar. * * The rule reference graph shows which rules reference which other rules. * The returned graph is inverted: it points from referee to referer. */ class RuleRefGraphFinder : public GrammarVisitor>> { public: RuleRefGraphFinder() = default; std::vector> Apply(const Grammar& grammar) { InitGrammar(grammar); rule_visit_graph_ = std::vector>(base_grammar_->NumRules()); for (int i = 0; i < static_cast(base_grammar_->NumRules()); ++i) { auto rule = base_grammar_->GetRule(i); auto grammar_expr = base_grammar_->GetGrammarExpr(rule.body_expr_id); cur_rule_id_ = i; VisitExpr(grammar_expr); } for (int i = 0; i < static_cast(base_grammar_->NumRules()); ++i) { std::sort(rule_visit_graph_[i].begin(), rule_visit_graph_[i].end()); auto end_it = std::unique(rule_visit_graph_[i].begin(), rule_visit_graph_[i].end()); rule_visit_graph_[i].erase(end_it, rule_visit_graph_[i].end()); } return std::move(rule_visit_graph_); } private: void VisitRuleRef(const GrammarExpr& grammar_expr) { rule_visit_graph_[grammar_expr[0]].push_back(cur_rule_id_); } void VisitRepeat(const GrammarExpr& grammar_expr) { rule_visit_graph_[grammar_expr[0]].push_back(cur_rule_id_); } void VisitTagDispatch(const GrammarExpr& grammar_expr) { auto tag_dispatch = base_grammar_->GetTagDispatch(grammar_expr); for (const auto& [trigger, rule_id] : tag_dispatch.tag_rule_pairs) { rule_visit_graph_[rule_id].push_back(cur_rule_id_); } } void VisitTokenTagDispatch(const GrammarExpr& grammar_expr) { auto ttd = base_grammar_->GetTokenTagDispatch(grammar_expr); for (const auto& [token_id, rule_id] : ttd.trigger_rule_pairs) { rule_visit_graph_[rule_id].push_back(cur_rule_id_); } } // Inversed reference graph: pointing from referee to referer std::vector> rule_visit_graph_; int32_t cur_rule_id_; }; /*! * \brief Analyzes which rules in a grammar can match the empty string. */ class AllowEmptyRuleAnalyzerImpl : public GrammarVisitor> { public: AllowEmptyRuleAnalyzerImpl() = default; std::vector Apply(const Grammar& grammar) final { InitGrammar(grammar); // Step 1: Find rules that explicitly allow empty string std::unordered_set empty_rule_id_set; FindExplicitEmptyRules(&empty_rule_id_set); // Step 2: Find rules that indirectly allow empty string. Using the Bellman-Ford algorithm // on the rule reference graph. std::vector> rule_ref_graph = RuleRefGraphFinder().Apply(grammar); FindIndirectEmptyRules(&empty_rule_id_set, rule_ref_graph); auto result = std::vector(empty_rule_id_set.begin(), empty_rule_id_set.end()); std::sort(result.begin(), result.end()); return result; } void FindExplicitEmptyRules(std::unordered_set* empty_rule_id_set) { for (int i = 0; i < static_cast(base_grammar_->NumRules()); ++i) { auto rule = base_grammar_->GetRule(i); auto grammar_expr = base_grammar_->GetGrammarExpr(rule.body_expr_id); if (grammar_expr.type == GrammarExprType::kTagDispatch || grammar_expr.type == GrammarExprType::kTokenTagDispatch) { empty_rule_id_set->insert(i); continue; } XGRAMMAR_DCHECK(grammar_expr.type == GrammarExprType::kChoices); if (base_grammar_->GetGrammarExpr(grammar_expr[0]).type == GrammarExprType::kEmptyStr) { empty_rule_id_set->insert(i); continue; } for (auto seq_id : grammar_expr) { auto seq_expr = base_grammar_->GetGrammarExpr(seq_id); if (std::all_of(seq_expr.begin(), seq_expr.end(), [&](int32_t i) { return base_grammar_->GetGrammarExpr(i).type == GrammarExprType::kCharacterClassStar; })) { empty_rule_id_set->insert(i); break; } } } } bool SeqExprIsEpsilon( const GrammarExpr& seq_expr, const std::unordered_set& empty_rule_id_set ) { if (seq_expr.type == GrammarExprType::kEmptyStr) { return true; } XGRAMMAR_DCHECK(seq_expr.type == GrammarExprType::kSequence); return std::all_of(seq_expr.begin(), seq_expr.end(), [&](int32_t i) { auto element_expr = base_grammar_->GetGrammarExpr(i); return (element_expr.type == GrammarExprType::kRuleRef && empty_rule_id_set.count(element_expr[0])) || element_expr.type == GrammarExprType::kCharacterClassStar || (element_expr.type == GrammarExprType::kRepeat && (empty_rule_id_set.count(element_expr[0]) || element_expr[1] == 0)); }); } void FindIndirectEmptyRules( std::unordered_set* empty_rule_id_set, const std::vector>& rule_ref_graph ) { std::queue queue; for (auto i : *empty_rule_id_set) { queue.push(i); } while (!queue.empty()) { auto rule_id = queue.front(); queue.pop(); XGRAMMAR_DCHECK(rule_id >= 0 && rule_id < static_cast(rule_ref_graph.size())); for (auto referer_rule_id : rule_ref_graph[rule_id]) { if (empty_rule_id_set->count(referer_rule_id)) { continue; } auto rule = base_grammar_->GetRule(referer_rule_id); auto grammar_expr = base_grammar_->GetGrammarExpr(rule.body_expr_id); XGRAMMAR_DCHECK( grammar_expr.type != GrammarExprType::kTagDispatch && grammar_expr.type != GrammarExprType::kTokenTagDispatch ) << "TagDispatch rules should already exist in empty_rule_id_set"; bool is_epsilon = std::any_of(grammar_expr.begin(), grammar_expr.end(), [&](int32_t i) { auto seq_expr = base_grammar_->GetGrammarExpr(i); return SeqExprIsEpsilon(seq_expr, *empty_rule_id_set); }); if (is_epsilon) { empty_rule_id_set->insert(referer_rule_id); queue.push(referer_rule_id); } } } } }; // Convert a Unicode codepoint to the packed UTF-8 format used by AddCharacterRange. // The packed format stores UTF-8 bytes as: (byte0 << 24) | (byte1 << 16) | (byte2 << 8) | byte3 // where byte0 is the first UTF-8 byte (leading byte) and subsequent bytes are continuation bytes. inline uint32_t CodepointToPackedUTF8(uint32_t codepoint) { if (codepoint <= 0x7F) { // 1-byte sequence (ASCII) return codepoint; } else if (codepoint <= 0x7FF) { // 2-byte sequence: byte0 = 110xxxxx, byte1 = 10xxxxxx uint8_t byte0 = 0xC0 | ((codepoint >> 6) & 0x1F); uint8_t byte1 = 0x80 | (codepoint & 0x3F); return (static_cast(byte0) << 8) | byte1; } else if (codepoint <= 0xFFFF) { // 3-byte sequence: byte0 = 1110xxxx, byte1 = 10xxxxxx, byte2 = 10xxxxxx uint8_t byte0 = 0xE0 | ((codepoint >> 12) & 0x0F); uint8_t byte1 = 0x80 | ((codepoint >> 6) & 0x3F); uint8_t byte2 = 0x80 | (codepoint & 0x3F); return (static_cast(byte0) << 16) | (static_cast(byte1) << 8) | byte2; } else { // 4-byte sequence: byte0 = 11110xxx, byte1-3 = 10xxxxxx uint8_t byte0 = 0xF0 | ((codepoint >> 18) & 0x07); uint8_t byte1 = 0x80 | ((codepoint >> 12) & 0x3F); uint8_t byte2 = 0x80 | ((codepoint >> 6) & 0x3F); uint8_t byte3 = 0x80 | (codepoint & 0x3F); return (static_cast(byte0) << 24) | (static_cast(byte1) << 16) | (static_cast(byte2) << 8) | byte3; } } class GrammarFSMBuilderImpl { public: const static uint32_t kMax1ByteUnicode = 0x7F; const static uint32_t kMin2BytesUnicode = 0xC080; const static uint32_t kMax2BytesUnicode = 0xDFBF; const static uint32_t kMin3BytesUnicode = 0xE08080; const static uint32_t kMax3BytesUnicode = 0xEFBFBF; const static uint32_t kMin4BytesUnicode = 0xF0808080; const static uint32_t kMax4BytesUnicode = 0xF7BFBFBF; void Apply(Grammar* grammar) { FSM complete_fsm; std::vector> per_rule_fsms((*grammar)->NumRules()); std::vector state_mapping; for (int i = 0; i < (*grammar)->NumRules(); ++i) { auto rule = (*grammar)->GetRule(i); auto grammar_expr = (*grammar)->GetGrammarExpr(rule.body_expr_id); if (grammar_expr.type == Grammar::Impl::GrammarExprType::kTagDispatch) { auto rule_fsm = TagDispatch((*grammar)->GetTagDispatch(grammar_expr)); XGRAMMAR_CHECK(rule_fsm.has_value()) << "Failed to build tag dispatch fsm for rule " << i; per_rule_fsms[i] = rule_fsm->AddToCompleteFSM(&complete_fsm, &state_mapping); } else if (grammar_expr.type == Grammar::Impl::GrammarExprType::kTokenTagDispatch) { auto rule_fsm = TokenTagDispatch((*grammar)->GetTokenTagDispatch(grammar_expr)); XGRAMMAR_CHECK(rule_fsm.has_value()) << "Failed to build token tag dispatch fsm for rule " << i; per_rule_fsms[i] = rule_fsm->AddToCompleteFSM(&complete_fsm, &state_mapping); } else { XGRAMMAR_DCHECK(grammar_expr.type == Grammar::Impl::GrammarExprType::kChoices); auto rule_fsm = Choices(grammar_expr, *grammar); if (rule_fsm.has_value()) { per_rule_fsms[i] = rule_fsm->AddToCompleteFSM(&complete_fsm, &state_mapping); } } } for (int i = 0; i < (*grammar)->NumRules(); ++i) { XGRAMMAR_DCHECK(per_rule_fsms[i].has_value()) << "Rule " << i << " (" << (*grammar)->GetRule(i).name << ") does not have an FSM after optimization"; } // Compress to compact fsm CompactFSM compact_complete_fsm = complete_fsm.ToCompact(); std::vector> compact_per_rule_fsms( (*grammar)->NumRules() ); for (int i = 0; i < (*grammar)->NumRules(); ++i) { if (per_rule_fsms[i]) { auto compact_fsm_with_se = CompactFSMWithStartEnd( compact_complete_fsm, per_rule_fsms[i]->GetFsm().GetStart(), per_rule_fsms[i]->GetFsm().GetEnds() ); compact_per_rule_fsms[i] = CompactFSMWithStartEndWithSize( compact_fsm_with_se, per_rule_fsms[i]->GetEdgeNum(), per_rule_fsms[i]->GetNodeNum() ); } } (*grammar)->complete_fsm = std::move(compact_complete_fsm); (*grammar)->per_rule_fsms = std::move(compact_per_rule_fsms); } /* Basic Building functions.*/ static FSMWithStartEnd RuleRef(const GrammarExpr& expr); static FSMWithStartEnd CharacterClass(const GrammarExpr& expr); static FSMWithStartEnd ByteString(const GrammarExpr& expr); static FSMWithStartEnd Repeat(const GrammarExpr& expr); static FSMWithStartEnd Token(const GrammarExpr& expr); static FSMWithStartEnd ExcludeToken(const GrammarExpr& expr); static std::optional TokenTagDispatch(const Grammar::Impl::TokenTagDispatch& ttd ); static std::optional Sequence(const GrammarExpr& expr, const Grammar& grammar); static std::optional Choices(const GrammarExpr& expr, const Grammar& grammar); static std::optional TagDispatch(const Grammar::Impl::TagDispatch& tag_dispatch); static void AddCharacterRange(FSMWithStartEnd& fsm, int from, int to, uint32_t min, uint32_t max); /* Building tool functions.*/ static std::optional BuildTagDispatch( const std::vector>& string_trigger_rules, bool loop_after_dispatch, const std::vector& excluded_strings ); static FSMWithStartEnd BuildNegativeCharacterClass(const GrammarExpr& expr); }; // This function will add a range [min, max] of characters to the FSM, and the length // of the characters are the same. void AddSameLengthCharacterRange( FSMWithStartEnd& fsm, int from, int to, uint32_t min, uint32_t max ) { uint8_t byte_min[4] = { static_cast(min & 0xFF), static_cast(min >> 8), static_cast(min >> 16), static_cast(min >> 24) }; uint8_t byte_max[4] = { static_cast(max & 0xFF), static_cast(max >> 8), static_cast(max >> 16), static_cast(max >> 24) }; // ASCII. if (byte_max[1] == 0) { fsm.GetFsm().AddEdge(from, to, byte_min[0], byte_max[0]); return; } if (byte_max[3] != 0) { // 4-byte unicode. if (byte_max[3] == byte_min[3]) { int tmp_state = fsm.AddState(); fsm.GetFsm().AddEdge(from, tmp_state, byte_min[3], byte_max[3]); min = (min & 0x00FFFFFF); max = (max & 0x00FFFFFF); AddSameLengthCharacterRange(fsm, tmp_state, to, min, max); return; } if ((min & 0x00FFFFFF) != 0x808080) { int tmp_state_min = fsm.AddState(); fsm.GetFsm().AddEdge(from, tmp_state_min, byte_min[3], byte_min[3]); AddSameLengthCharacterRange(fsm, tmp_state_min, to, (min & 0x00FFFFFF), 0x00BFBFBF); } else { byte_min[3]--; } if ((max & 0x00FFFFFF) != 0xBFBFBF) { int tmp_state_max = fsm.AddState(); fsm.GetFsm().AddEdge(from, tmp_state_max, byte_max[3], byte_max[3]); AddSameLengthCharacterRange(fsm, tmp_state_max, to, 0x00808080, (max & 0x00FFFFFF)); } else { byte_max[3]++; } if (byte_max[3] - byte_min[3] > 1) { int tmp_state_mid = fsm.AddState(); // First byte. fsm.GetFsm().AddEdge(from, tmp_state_mid, byte_min[3] + 1, byte_max[3] - 1); int tmp_state_mid2 = fsm.AddState(); // Second byte. fsm.GetFsm().AddEdge(tmp_state_mid, tmp_state_mid2, 0x80, 0xBF); int tmp_state_mid3 = fsm.AddState(); // Third byte. fsm.GetFsm().AddEdge(tmp_state_mid2, tmp_state_mid3, 0x80, 0xBF); // Last byte. fsm.GetFsm().AddEdge(tmp_state_mid3, to, 0x80, 0xBF); } return; } if (byte_max[2] != 0) { // 3 byte unicode. if (byte_max[2] == byte_min[2]) { int tmp_state = fsm.AddState(); fsm.GetFsm().AddEdge(from, tmp_state, byte_min[2], byte_max[2]); min = (min & 0x00FFFF); max = (max & 0x00FFFF); AddSameLengthCharacterRange(fsm, tmp_state, to, min, max); return; } if ((min & 0x00FFFF) != 0x8080) { int tmp_state_min = fsm.AddState(); fsm.GetFsm().AddEdge(from, tmp_state_min, byte_min[2], byte_min[2]); AddSameLengthCharacterRange(fsm, tmp_state_min, to, (min & 0x00FFFF), 0x00BFBF); } else { byte_min[2]--; } if ((max & 0x00FFFF) != 0xBFBF) { int tmp_state_max = fsm.AddState(); fsm.GetFsm().AddEdge(from, tmp_state_max, byte_max[2], byte_max[2]); AddSameLengthCharacterRange(fsm, tmp_state_max, to, 0x0080, (max & 0x00FFFF)); } else { byte_max[2]++; } if (byte_max[2] - byte_min[2] > 1) { int tmp_state_mid = fsm.AddState(); // First byte. fsm.GetFsm().AddEdge(from, tmp_state_mid, byte_min[2] + 1, byte_max[2] - 1); int tmp_state_mid2 = fsm.AddState(); // Second byte. fsm.GetFsm().AddEdge(tmp_state_mid, tmp_state_mid2, 0x80, 0xBF); // Last byte. fsm.GetFsm().AddEdge(tmp_state_mid2, to, 0x80, 0xBF); } return; } // 2 byte unicode. if (byte_max[1] == byte_min[1]) { int tmp_state = fsm.AddState(); fsm.GetFsm().AddEdge(from, tmp_state, byte_min[1], byte_max[1]); min = (min & 0x00FF); max = (max & 0x00FF); AddSameLengthCharacterRange(fsm, tmp_state, to, min, max); return; } if ((min & 0x00FF) != 0x80) { int tmp_state_min = fsm.AddState(); fsm.GetFsm().AddEdge(from, tmp_state_min, byte_min[1], byte_min[1]); AddSameLengthCharacterRange(fsm, tmp_state_min, to, (min & 0x00FF), 0x00BF); } else { byte_min[1]--; } if ((max & 0x00FF) != 0xBF) { int tmp_state_max = fsm.AddState(); fsm.GetFsm().AddEdge(from, tmp_state_max, byte_max[1], byte_max[1]); AddSameLengthCharacterRange(fsm, tmp_state_max, to, 0x0080, (max & 0x00FF)); } else { byte_max[1]++; } if (byte_max[1] - byte_min[1] > 1) { int tmp_state_mid = fsm.AddState(); // First byte. fsm.GetFsm().AddEdge(from, tmp_state_mid, byte_min[1] + 1, byte_max[1] - 1); fsm.GetFsm().AddEdge(tmp_state_mid, to, 0x80, 0xBF); } return; } // This function will add a range [min, max] of unicode characters to the FSM. void GrammarFSMBuilderImpl::AddCharacterRange( FSMWithStartEnd& fsm, int from, int to, uint32_t min, uint32_t max ) { XGRAMMAR_CHECK(min <= max) << "Invalid character range: min (" << min << ") > max (" << max << ")"; // Ensure max and min are valid unicode value. if (max > kMax4BytesUnicode) { max = kMax4BytesUnicode; } else if (max > kMax3BytesUnicode) { if (max < kMin4BytesUnicode) { max = kMax3BytesUnicode; } } else if (max > kMax2BytesUnicode) { if (max < kMin3BytesUnicode) { max = kMax2BytesUnicode; } } else if (max < kMin2BytesUnicode && (max > kMax1ByteUnicode)) { max = kMax1ByteUnicode; } if (min > kMax4BytesUnicode) { min = kMax4BytesUnicode; } else if (min > kMax3BytesUnicode) { if (min < kMin4BytesUnicode) { min = kMin4BytesUnicode; } } else if (min > kMax2BytesUnicode) { if (min < kMin3BytesUnicode) { min = kMin3BytesUnicode; } } else if (min < kMin2BytesUnicode && (min > kMax1ByteUnicode)) { min = kMin2BytesUnicode; } // Step2. Divide the range into several ranges, which contain characters with different lengths. if (max <= kMax1ByteUnicode) { AddSameLengthCharacterRange(fsm, from, to, min, max); return; } if (max <= kMax2BytesUnicode) { if (min >= kMin2BytesUnicode) { AddSameLengthCharacterRange(fsm, from, to, min, max); } else { AddSameLengthCharacterRange(fsm, from, to, min, kMax1ByteUnicode); AddSameLengthCharacterRange(fsm, from, to, kMin2BytesUnicode, max); } return; } if (max <= kMax3BytesUnicode) { if (min >= kMin3BytesUnicode) { AddSameLengthCharacterRange(fsm, from, to, min, max); } else if (min >= kMin2BytesUnicode) { AddSameLengthCharacterRange(fsm, from, to, min, kMax2BytesUnicode); AddSameLengthCharacterRange(fsm, from, to, kMin3BytesUnicode, max); } else { AddSameLengthCharacterRange(fsm, from, to, min, kMax1ByteUnicode); AddSameLengthCharacterRange(fsm, from, to, kMin2BytesUnicode, kMax2BytesUnicode); AddSameLengthCharacterRange(fsm, from, to, kMin3BytesUnicode, max); } return; } XGRAMMAR_CHECK(max <= kMax4BytesUnicode); if (min >= kMin4BytesUnicode) { AddSameLengthCharacterRange(fsm, from, to, min, max); } else if (min >= kMin3BytesUnicode) { AddSameLengthCharacterRange(fsm, from, to, min, kMax3BytesUnicode); AddSameLengthCharacterRange(fsm, from, to, kMin4BytesUnicode, max); } else if (min >= kMin2BytesUnicode) { AddSameLengthCharacterRange(fsm, from, to, min, kMax2BytesUnicode); AddSameLengthCharacterRange(fsm, from, to, kMin3BytesUnicode, kMax3BytesUnicode); AddSameLengthCharacterRange(fsm, from, to, kMin4BytesUnicode, max); } else { AddSameLengthCharacterRange(fsm, from, to, min, kMax1ByteUnicode); AddSameLengthCharacterRange(fsm, from, to, kMin2BytesUnicode, kMax2BytesUnicode); AddSameLengthCharacterRange(fsm, from, to, kMin3BytesUnicode, kMax3BytesUnicode); AddSameLengthCharacterRange(fsm, from, to, kMin4BytesUnicode, max); } return; } FSMWithStartEnd GrammarFSMBuilderImpl::BuildNegativeCharacterClass(const GrammarExpr& expr) { XGRAMMAR_DCHECK( expr.type == ExprType::kCharacterClass || expr.type == ExprType::kCharacterClassStar ); XGRAMMAR_DCHECK(expr[0]); // Negative character class should be true. std::bitset<128> char_set; for (int i = 1; i < static_cast(expr.size()); i += 2) { uint8_t byte_min = static_cast(expr[i]); uint8_t byte_max = static_cast(expr[i + 1]); if (byte_max > 128) { XGRAMMAR_LOG(WARNING) << "Negative Character class contains byte greater than 127, " << "clamping to 127."; byte_max = 127; } for (uint8_t j = byte_min; j <= byte_max; ++j) { char_set.set(j); } } // Construct the basic FSM. FSMWithStartEnd result_fsm; int start_state = result_fsm.AddState(); bool is_star = expr.type == ExprType::kCharacterClassStar; result_fsm.SetStartState(start_state); int end_state = -1; if (is_star) { end_state = start_state; } else { end_state = result_fsm.AddState(); } result_fsm.AddEndState(end_state); int left_bound = -1; for (int i = 0; i < 128; ++i) { if (!char_set[i]) { left_bound = i; int right_bound = i + 1; while (right_bound < 128 && !char_set[right_bound]) { right_bound++; } result_fsm.GetFsm().AddEdge( start_state, end_state, static_cast(left_bound), static_cast(right_bound - 1) ); i = right_bound; } } AddCharacterRange(result_fsm, start_state, end_state, kMin2BytesUnicode, kMax4BytesUnicode); return result_fsm; } FSMWithStartEnd GrammarFSMBuilderImpl::CharacterClass(const GrammarExpr& expr) { bool is_negative = expr[0]; FSMWithStartEnd result_fsm; if (is_negative) { result_fsm = BuildNegativeCharacterClass(expr); return result_fsm; } int start_state = result_fsm.AddState(); result_fsm.SetStartState(start_state); bool is_star = expr.type == ExprType::kCharacterClassStar; int end_state = -1; if (is_star) { end_state = start_state; } else { end_state = result_fsm.AddState(); } result_fsm.AddEndState(end_state); for (int i = 1; i < static_cast(expr.size()); i += 2) { uint32_t codepoint_min = static_cast(expr[i]); uint32_t codepoint_max = static_cast(expr[i + 1]); // Convert Unicode codepoints to packed UTF-8 format for AddCharacterRange uint32_t packed_min = CodepointToPackedUTF8(codepoint_min); uint32_t packed_max = CodepointToPackedUTF8(codepoint_max); AddCharacterRange(result_fsm, start_state, end_state, packed_min, packed_max); } return result_fsm; } FSMWithStartEnd GrammarFSMBuilderImpl::Repeat(const GrammarExpr& expr) { int32_t rule_id = expr[0]; int32_t lower = expr[1]; int32_t upper = expr[2]; FSMWithStartEnd repeat_fsm; repeat_fsm.AddState(); repeat_fsm.AddState(); repeat_fsm.SetStartState(0); repeat_fsm.AddEndState(1); repeat_fsm.GetFsm().AddRepeatEdge(0, 1, rule_id, lower, upper); return repeat_fsm; } FSMWithStartEnd GrammarFSMBuilderImpl::Token(const GrammarExpr& expr) { XGRAMMAR_DCHECK(expr.type == ExprType::kToken); std::vector token_ids(expr.begin(), expr.end()); FSM fsm(2); fsm.AddTokenEdge(0, 1, token_ids); return FSMWithStartEnd(fsm, 0, {false, true}); } FSMWithStartEnd GrammarFSMBuilderImpl::ExcludeToken(const GrammarExpr& expr) { XGRAMMAR_DCHECK(expr.type == ExprType::kExcludeToken); std::vector token_ids(expr.begin(), expr.end()); FSM fsm(2); fsm.AddExcludeTokenEdge(0, 1, token_ids); return FSMWithStartEnd(fsm, 0, {false, true}); } std::optional GrammarFSMBuilderImpl::TokenTagDispatch( const Grammar::Impl::TokenTagDispatch& ttd ) { int num_triggers = static_cast(ttd.trigger_rule_pairs.size()); bool loop = ttd.loop_after_dispatch; int num_states = 1 + num_triggers + (loop ? 0 : 1); FSM fsm(num_states); std::vector ends(num_states, false); int start = 0; ends[start] = true; int end_state = -1; if (!loop) { end_state = num_states - 1; ends[end_state] = true; } std::vector self_loop_exclude; for (const auto& [token_id, rule_id] : ttd.trigger_rule_pairs) { self_loop_exclude.push_back(token_id); } for (auto excl_id : ttd.excludes) { self_loop_exclude.push_back(excl_id); } std::sort(self_loop_exclude.begin(), self_loop_exclude.end()); self_loop_exclude.erase( std::unique(self_loop_exclude.begin(), self_loop_exclude.end()), self_loop_exclude.end() ); for (int i = 0; i < num_triggers; ++i) { int dispatch_state = 1 + i; auto [token_id, rule_id] = ttd.trigger_rule_pairs[i]; fsm.AddTokenEdge(start, dispatch_state, {token_id}); int target = loop ? start : end_state; fsm.AddRuleEdge(dispatch_state, target, static_cast(rule_id)); } fsm.AddExcludeTokenEdge(start, start, self_loop_exclude); return FSMWithStartEnd(fsm, start, ends); } std::optional GrammarFSMBuilderImpl::Sequence( const GrammarExpr& expr, const Grammar& grammar ) { std::vector fsm_lists; // Build the fsm of sub-expressions. for (const auto& sequence_id : expr) { const auto& sequence_expr = grammar->GetGrammarExpr(sequence_id); switch (sequence_expr.type) { case (ExprType::kByteString): { fsm_lists.push_back(ByteString(sequence_expr)); break; } case (ExprType::kRuleRef): { fsm_lists.push_back(RuleRef(sequence_expr)); break; } case (ExprType::kCharacterClass): case (ExprType::kCharacterClassStar): { fsm_lists.push_back(CharacterClass(sequence_expr)); break; } case (ExprType::kRepeat): { fsm_lists.push_back(Repeat(sequence_expr)); break; } case (ExprType::kToken): { fsm_lists.push_back(Token(sequence_expr)); break; } case (ExprType::kExcludeToken): { fsm_lists.push_back(ExcludeToken(sequence_expr)); break; } default: { return std::nullopt; } } } // Check if the sequence is empty. if (fsm_lists.empty()) { FSMWithStartEnd empty_fsm; empty_fsm.AddState(); empty_fsm.SetStartState(0); empty_fsm.AddEndState(0); return empty_fsm; } return FSMWithStartEnd::Concat(fsm_lists); } FSMWithStartEnd GrammarFSMBuilderImpl::RuleRef(const GrammarExpr& expr) { FSMWithStartEnd result_fsm; result_fsm.AddState(); result_fsm.AddState(); result_fsm.SetStartState(0); result_fsm.AddEndState(1); result_fsm.GetFsm().AddRuleEdge(0, 1, expr[0]); return result_fsm; } FSMWithStartEnd GrammarFSMBuilderImpl::ByteString(const GrammarExpr& expr) { XGRAMMAR_DCHECK(expr.type == ExprType::kByteString); FSMWithStartEnd result_fsm; int current_state = result_fsm.AddState(); result_fsm.SetStartState(current_state); for (const auto& byte : expr) { int next_state = result_fsm.AddState(); result_fsm.GetFsm().AddEdge( current_state, next_state, static_cast(byte), static_cast(byte) ); current_state = next_state; } result_fsm.AddEndState(current_state); return result_fsm; } std::optional GrammarFSMBuilderImpl::Choices( const GrammarExpr& expr, const Grammar& grammar ) { XGRAMMAR_DCHECK(expr.type == ExprType::kChoices); std::vector fsm_list; bool nullable = false; for (const auto& choice_id : expr) { const auto& choice_expr = grammar->GetGrammarExpr(choice_id); if (choice_expr.type == ExprType::kEmptyStr) { nullable = true; continue; } XGRAMMAR_DCHECK(choice_expr.type == ExprType::kSequence); auto fsm_result = Sequence(choice_expr, grammar); if (!fsm_result.has_value()) { return std::nullopt; } fsm_list.push_back(std::move(fsm_result.value())); } if (fsm_list.empty()) { // It's an empty rule. FSMWithStartEnd empty_fsm; empty_fsm.AddState(); empty_fsm.SetStartState(0); empty_fsm.AddEndState(0); return empty_fsm; } if (nullable) { FSMWithStartEnd null_fsm; null_fsm.AddState(); null_fsm.SetStartState(0); null_fsm.AddEndState(0); fsm_list.push_back(std::move(null_fsm)); } auto result = FSMWithStartEnd::Union(fsm_list); result = result.SimplifyEpsilon(); result = result.MergeEquivalentStates(); return result; } std::optional GrammarFSMBuilderImpl::BuildTagDispatch( const std::vector>& string_trigger_rules, bool loop_after_dispatch, const std::vector& excluded_strings ) { std::vector tag_names; tag_names.reserve(string_trigger_rules.size()); for (const auto& [tag_name, tag_id] : string_trigger_rules) { tag_names.push_back(tag_name); } std::vector end_states; auto trie_result = TrieFSMBuilder::Build(tag_names, excluded_strings, &end_states, true, true); if (!trie_result.has_value()) { return std::nullopt; } auto trie_fsm = trie_result->GetFsm(); auto start = trie_result->GetStart(); std::unordered_set old_ends; std::vector ends(trie_fsm.NumStates(), false); for (int end = 0; end < trie_result->NumStates(); end++) { if (trie_result->IsEndState(end)) { old_ends.insert(end); } } // The final end states are all but old_ends. for (int i = 0; i < trie_fsm.NumStates(); i++) { if (old_ends.count(i) == 0) { ends[i] = true; } } // Add rule ref edges for string triggers for (int i = 0; i < static_cast(string_trigger_rules.size()); i++) { int next_state; if (loop_after_dispatch) { next_state = start; } else { next_state = trie_fsm.AddState(); ends.push_back(true); } trie_fsm.AddRuleEdge(end_states[i], next_state, string_trigger_rules[i].second); } return FSMWithStartEnd(trie_fsm, start, ends); } std::optional GrammarFSMBuilderImpl::TagDispatch( const Grammar::Impl::TagDispatch& tag_dispatch ) { std::vector> string_trigger_rules( tag_dispatch.tag_rule_pairs.begin(), tag_dispatch.tag_rule_pairs.end() ); return BuildTagDispatch( string_trigger_rules, tag_dispatch.loop_after_dispatch, tag_dispatch.excludes ); } class RepetitionRangeExpanderImpl : public GrammarMutator { public: using GrammarMutator::Apply; using GrammarMutator::GrammarMutator; private: int32_t VisitRepeat(const GrammarExpr& grammar_expr) final { int32_t ref_rule_id = grammar_expr[0]; int64_t lower = grammar_expr[1]; int64_t upper = grammar_expr[2]; return HandleRepetitionRange(cur_rule_name_, ref_rule_id, lower, upper); } /*! * \brief Handle repetition range by unzipping into explicit sequence/choice (for small bounds). * \param cur_rule_name Name hint for generated rules. * \param grammar_expr_id The expression to repeat. * \param lower Minimum count (inclusive). * \param upper Maximum count (inclusive), or -1 for unbounded. * \return grammar_expr_id of the repetition result. */ int32_t LegacyHandleRepetitionRange( const std::string& cur_rule_name, int32_t grammar_expr_id, int64_t lower, int64_t upper ); /*! * \brief Handle repetition range {lower, upper}, using unzip for small bounds or kRepeat for * large. * \param cur_rule_name Name hint for generated rules. * \param rule_id The rule to repeat. * \param lower Minimum count (inclusive). * \param upper Maximum count (inclusive), or -1 for unbounded. * \return grammar_expr_id of the repetition result. */ int32_t HandleRepetitionRange( const std::string& cur_rule_name, int32_t rule_id, int64_t lower, int64_t upper ); }; /****************** Repetition range helpers ******************/ int32_t RepetitionRangeExpanderImpl::LegacyHandleRepetitionRange( const std::string& cur_rule_name, int32_t grammar_expr_id, int64_t lower, int64_t upper ) { // Construct expr expr ... expr (l times) std::vector elements; for (int64_t i = 0; i < lower; ++i) { elements.push_back(grammar_expr_id); } // Case 1: {l}: // expr expr ... expr (l times) if (upper == lower) { auto result_rule_id = builder_->AddRuleWithHint( cur_rule_name, builder_->AddChoices({builder_->AddSequence(elements)}) ); return builder_->AddRuleRef(result_rule_id); } // Case 2: {l,}: // expr expr ... expr (l times) rest // rest ::= "" | expr rest if (upper == -1) { auto new_rule_name = builder_->GetNewRuleName(cur_rule_name); auto new_rule_id = builder_->AddEmptyRule(new_rule_name); auto ref_to_new_rule = builder_->AddRuleRef(new_rule_id); auto new_grammar_expr_id = builder_->AddChoices( {builder_->AddEmptyStr(), builder_->AddSequence({grammar_expr_id, ref_to_new_rule})} ); builder_->UpdateRuleBody(new_rule_id, new_grammar_expr_id); elements.push_back(builder_->AddRuleRef(new_rule_id)); auto result_rule_id = builder_->AddRuleWithHint( cur_rule_name, builder_->AddChoices({builder_->AddSequence(elements)}) ); return builder_->AddRuleRef(result_rule_id); } // Case 3: {l, r} (r - l >= 1) // expr expr ... expr (l times) rest1 // rest1 ::= "" | expr rest2 // rest2 ::= "" | expr rest3 // ... // rest(r - l) ::= "" | expr std::vector rest_rule_ids; for (int64_t i = 0; i < upper - lower; ++i) { auto new_rule_name = builder_->GetNewRuleName(cur_rule_name); rest_rule_ids.push_back(builder_->AddEmptyRule(new_rule_name)); } for (int64_t i = 0; i < upper - lower - 1; ++i) { auto ref_to_next_rule = builder_->AddRuleRef(rest_rule_ids[i + 1]); auto new_grammar_expr_id = builder_->AddChoices( {builder_->AddEmptyStr(), builder_->AddSequence({grammar_expr_id, ref_to_next_rule})} ); builder_->UpdateRuleBody(rest_rule_ids[i], new_grammar_expr_id); } auto last_grammar_expr_id = builder_->AddChoices({builder_->AddEmptyStr(), builder_->AddSequence({grammar_expr_id})}); builder_->UpdateRuleBody(rest_rule_ids.back(), last_grammar_expr_id); elements.push_back(builder_->AddRuleRef(rest_rule_ids[0])); auto result_rule_id = builder_->AddRuleWithHint( cur_rule_name, builder_->AddChoices({builder_->AddSequence(elements)}) ); return builder_->AddRuleRef(result_rule_id); } int32_t RepetitionRangeExpanderImpl::HandleRepetitionRange( const std::string& cur_rule_name, int32_t rule_id, int64_t lower, int64_t upper ) { static const int64_t kUnzipThreshold = 128; XGRAMMAR_DCHECK(lower >= 0); XGRAMMAR_DCHECK(upper == -1 || upper >= lower); // Check if the referred rule is only one single element. If so, we can directly use the element // for further optimization. int32_t grammar_expr_id = builder_->AddRuleRef(rule_id); const auto& ref_rule = base_grammar_->GetRule(rule_id); const auto& ref_rule_body = base_grammar_->GetGrammarExpr(ref_rule.body_expr_id); if (ref_rule_body.type == GrammarBuilder::GrammarExprType::kChoices && ref_rule_body.size() == 1) { const auto& ref_choice = base_grammar_->GetGrammarExpr(ref_rule_body[0]); if (ref_choice.size() == 1) { grammar_expr_id = builder_->AddGrammarExpr(base_grammar_->GetGrammarExpr(ref_choice[0])); } } // Case 1.1 small upper (<=threshold), unzip the repetition. // Case 1.2 unbounded upper, and lower is also small (<=threshold), unzip the lower part. if ((upper != -1 && upper <= kUnzipThreshold) || (upper == -1 && lower <= kUnzipThreshold)) { return LegacyHandleRepetitionRange(cur_rule_name, grammar_expr_id, lower, upper); } // Case 2. upper is unbounded, and lower is large (>threshold). // Or upper is bounded, but upper > threshold. // Case 2.1.1. lower is smaller than threshold, and upper is large. Transform {lower, upper} into: // {threshold, upper} | {lower, threshold} std::vector choices; if (lower < kUnzipThreshold) { choices.push_back(builder_->AddSequence( {LegacyHandleRepetitionRange(cur_rule_name, grammar_expr_id, lower, kUnzipThreshold - 1)} )); lower = kUnzipThreshold; } std::optional infinite_repetition_id = std::nullopt; std::vector repeated_sequence; // Now, we transform {lower, upper} into {max{threshold, lower}, upper}. // Case 2.2 upper is unbounded. We will transform it into {lower} {0, inf}. if (upper == -1) { const auto& rule_expr = builder_->GetGrammarExpr(grammar_expr_id); if (rule_expr.type == GrammarBuilder::GrammarExprType::kCharacterClass) { std::vector character_ranges; bool is_negative = rule_expr[0]; for (int i = 1; i < static_cast(rule_expr.size()); i += 2) { character_ranges.push_back({rule_expr[i], rule_expr[i + 1]}); } infinite_repetition_id = builder_->AddCharacterClassStar(character_ranges, is_negative); } else { const auto unbounded_rule_id = builder_->AddEmptyRule(builder_->GetNewRuleName(cur_rule_name + "_repeat_inf")); int recursion_sequence = builder_->AddSequence({grammar_expr_id, builder_->AddRuleRef(unbounded_rule_id)}); int recursion_choice = builder_->AddChoices({builder_->AddEmptyStr(), recursion_sequence}); builder_->UpdateRuleBody(unbounded_rule_id, recursion_choice); infinite_repetition_id = builder_->AddRuleRef(unbounded_rule_id); } upper = lower; } // Handle the {lower, upper} part, where threshold <= lower <= upper. const auto repeat_name = cur_rule_name + "_repeat_1"; XGRAMMAR_DCHECK(lower >= kUnzipThreshold && upper >= lower); // If we have infinite repetition part, add it to the sequence. if (infinite_repetition_id.has_value()) { repeated_sequence.push_back(infinite_repetition_id.value()); } // The repetition body. if (upper != kUnzipThreshold) { XGRAMMAR_DCHECK(upper > kUnzipThreshold); auto new_grammar_expr_id = builder_->AddChoices({builder_->AddSequence({grammar_expr_id})}); auto new_rule_id = builder_->AddRuleWithHint(repeat_name, new_grammar_expr_id); auto new_repeated_ref_rule_expr = builder_->AddChoices({builder_->AddSequence( {builder_->AddRepeat(new_rule_id, lower - kUnzipThreshold, upper - kUnzipThreshold)} )}); auto new_repeated_rule_id = builder_->AddRuleWithHint(repeat_name + "_inner", new_repeated_ref_rule_expr); repeated_sequence.push_back(builder_->AddRuleRef(new_repeated_rule_id)); std::vector repetition_lookahead(kUnzipThreshold, grammar_expr_id); builder_->UpdateLookaheadAssertion(new_rule_id, builder_->AddSequence(repetition_lookahead)); } // Add the last threshold grammar_expr_id to the sequence. for (int i = 0; i < kUnzipThreshold; ++i) { repeated_sequence.push_back(grammar_expr_id); } // Add the sequence to choices. choices.push_back(builder_->AddSequence(repeated_sequence)); auto result_rule_id = builder_->AddRuleWithHint(cur_rule_name, builder_->AddChoices(choices)); return builder_->AddRuleRef(result_rule_id); } class RepetitionNormalizerImpl { public: void Apply(Grammar* grammar) { auto& grammar_ref = *grammar; for (int i = 0; i < grammar_ref->NumGrammarExprs(); ++i) { auto expr = grammar_ref->GetGrammarExpr(i); if (expr.type != Grammar::Impl::GrammarExprType::kRepeat) { continue; } int repeat_rule_id = expr[0]; grammar_ref->GetRule(repeat_rule_id).is_exact_lookahead = true; if (std::binary_search( grammar_ref->allow_empty_rule_ids.begin(), grammar_ref->allow_empty_rule_ids.end(), repeat_rule_id )) { // The repeated rule can be empty, so we need to normalize it. expr.SetData(1, 0); // Set min repeat to 0 } } } }; class GrammarOptimizerImpl { public: static Grammar Apply(const Grammar& grammar) { auto result = ByteStringFuser::Apply(grammar); result = RuleInliner::Apply(result); result = RepetitionRangeExpander::Apply(result); result = DeadCodeEliminator::Apply(result); result = LookaheadAssertionAnalyzer::Apply(result); result->allow_empty_rule_ids = AllowEmptyRuleAnalyzer::Apply(result); RepetitionNormalizer::Apply(&result); GrammarFSMBuilder::Apply(&result); result->optimized = true; return result; } }; class ByteStringFuserImpl : public GrammarMutator { public: using GrammarMutator::Apply; using GrammarMutator::GrammarMutator; private: /*! * \brief Visit a GrammarExpr containing a sequence. * \returns A list of new sequence GrammarExpr ids. */ int32_t VisitSequence(const GrammarExpr& grammar_expr) final { std::vector new_sequence_ids; std::vector cur_byte_string; for (auto i : grammar_expr) { auto element_expr = base_grammar_->GetGrammarExpr(i); if (element_expr.type == GrammarExprType::kByteString) { cur_byte_string.insert(cur_byte_string.end(), element_expr.begin(), element_expr.end()); continue; } else { if (!cur_byte_string.empty()) { new_sequence_ids.push_back(builder_->AddByteString(cur_byte_string)); cur_byte_string.clear(); } new_sequence_ids.push_back(builder_->AddGrammarExpr(element_expr)); } } if (!cur_byte_string.empty()) { new_sequence_ids.push_back(builder_->AddByteString(cur_byte_string)); } return builder_->AddSequence(new_sequence_ids); } }; class RootRuleRenamerImpl { public: static Grammar Apply(const Grammar& grammar) { // If the root name is "root", return directly. if (grammar->GetRootRule().name == "root") { return grammar; } // Collect all the rule names. std::unordered_set rule_names; int root_name_rule_id = -1; for (int i = 0; i < grammar->NumRules(); i++) { const auto& rule_name = grammar->GetRule(i).name; if (rule_name == "root") { root_name_rule_id = i; } rule_names.insert(rule_name); } // Rename the rules. Grammar grammar_copy = grammar; grammar_copy->GetRule(grammar_copy->GetRootRuleId()).name = "root"; if (root_name_rule_id != -1) { std::string rule_prefix = "root_"; for (int i = 0; i <= grammar_copy->NumRules(); i++) { std::string new_rule_name = rule_prefix + std::to_string(i); if (rule_names.find(new_rule_name) == rule_names.end()) { grammar_copy->GetRule(root_name_rule_id).name = new_rule_name; break; } XGRAMMAR_DCHECK(false ) << "The rule must be renamed successfully after (n + 1) times of iterations."; } } return grammar_copy; } }; class GrammarFSMHasherImpl { public: void Apply(Grammar* grammar); static std::optional HashSequence(const Grammar& grammar, int32_t sequence_id); static constexpr int16_t kNotEndStateFlag = -0x100; static constexpr int16_t kEndStateFlag = -0x200; static constexpr int16_t kSelfRecursionFlag = -0x300; static constexpr int16_t kSimpleCycleFlag = -0x400; static constexpr int16_t kUnKnownFlag = -0x500; private: Grammar* grammar_; std::vector visited_; std::vector> ref_graph_from_referrer_to_referee_; std::vector> ref_graph_from_referee_to_referrer_; std::vector> sorted_edges_; std::vector has_inward_edges_; /*! * \brief Get the hash value of a fsm, with a given grammar. */ uint64_t HashFsm(int fsm_index); /*! * \brief Find a simple cycle in the reference graph, And hash the * fsms in the simple cycle. */ bool FindSimpleCycle(); /*! * \brief Hash the fsms in the simple cycle. */ void HashSimpleCycle(const std::vector& simple_cycle); /*! * \brief Find a simple fsm that can be hashed. If it can't, it will * call FindSimpleCycle() and try to simplify the graph, and then try to * find a simple fsm again. */ int32_t FindSimpleFsmCanBeHashed(); std::pair IsPartialHashable(int fsm_index); }; bool GrammarFSMHasherImpl::FindSimpleCycle() { // Try to find a simple cycle. std::vector not_simple_cycle = visited_; for (size_t i = 0; i < ref_graph_from_referee_to_referrer_.size(); i++) { if (not_simple_cycle[i]) { continue; } // Not a simple cycle if it has more than one referee. std::stack dfs_stack; std::vector simple_cycle; auto in_stack = std::vector(ref_graph_from_referee_to_referrer_.size(), false); dfs_stack.push(static_cast(i)); int32_t current_fsm_index = i; in_stack[current_fsm_index] = true; while ((ref_graph_from_referrer_to_referee_[current_fsm_index].size() == 1) && !not_simple_cycle[current_fsm_index]) { XGRAMMAR_CHECK(current_fsm_index != ref_graph_from_referrer_to_referee_[current_fsm_index][0]) << "Self-recursion cycle found in the reference graph, which is not allowed."; not_simple_cycle[current_fsm_index] = true; current_fsm_index = ref_graph_from_referrer_to_referee_[current_fsm_index][0]; if (in_stack[current_fsm_index]) { simple_cycle.push_back(current_fsm_index); while (dfs_stack.top() != current_fsm_index) { simple_cycle.push_back(dfs_stack.top()); dfs_stack.pop(); } // Found a simple cycle. break; } else { dfs_stack.push(current_fsm_index); in_stack[current_fsm_index] = true; } } if (!simple_cycle.empty()) { HashSimpleCycle(simple_cycle); return true; } } return false; } void GrammarFSMHasherImpl::HashSimpleCycle(const std::vector& simple_cycle) { // Initialize the cycle hash. for (const auto& cycle_id : simple_cycle) { visited_[cycle_id] = true; grammar_->ImplPtr()->per_rule_fsm_hashes[cycle_id] = kSimpleCycleFlag; } std::vector local_cycle_hash; local_cycle_hash.reserve(simple_cycle.size()); for (const auto& cycle_id : simple_cycle) { local_cycle_hash.push_back(HashFsm(cycle_id)); } std::vector local_cycle_hash_copy = local_cycle_hash; for (int i = 0; i < static_cast(local_cycle_hash.size()); i++) { uint64_t current_hash = 0; for (int j = 0; j < static_cast(local_cycle_hash.size()); j++) { current_hash = HashCombine(current_hash, local_cycle_hash_copy[(i + j) % local_cycle_hash.size()]); } local_cycle_hash[i] = current_hash; } for (int i = 0; i < static_cast(simple_cycle.size()); i++) { grammar_->ImplPtr()->per_rule_fsm_hashes[simple_cycle[i]] = local_cycle_hash[i]; for (const auto& referer : ref_graph_from_referee_to_referrer_[simple_cycle[i]]) { ref_graph_from_referrer_to_referee_[referer].erase(std::find_if( ref_graph_from_referrer_to_referee_[referer].begin(), ref_graph_from_referrer_to_referee_[referer].end(), [&](int32_t rule_id) { return rule_id == simple_cycle[i]; } )); } } } int32_t GrammarFSMHasherImpl::FindSimpleFsmCanBeHashed() { bool possible_to_find = true; while (possible_to_find) { possible_to_find = false; for (size_t i = 0; i < ref_graph_from_referrer_to_referee_.size(); i++) { if (visited_[i]) { continue; } if (ref_graph_from_referrer_to_referee_[i].empty()) { return i; } if (ref_graph_from_referrer_to_referee_[i].size() == 1 && ref_graph_from_referrer_to_referee_[i][0] == static_cast(i)) { // Self-recursion fsm. return static_cast(i); } } // Try to find a simple cycle. We must ensure there are not self-recursion cycles. possible_to_find = FindSimpleCycle(); } return -1; } void GrammarFSMHasherImpl::Apply(Grammar* grammar) { grammar_ = grammar; grammar->ImplPtr()->per_rule_fsm_hashes = std::vector>((*grammar)->NumRules()); grammar->ImplPtr()->per_rule_fsm_new_state_ids.resize((*grammar)->NumRules()); ref_graph_from_referee_to_referrer_.clear(); ref_graph_from_referrer_to_referee_.clear(); sorted_edges_.clear(); visited_ = std::vector((*grammar)->NumRules(), false); has_inward_edges_ = std::vector((*grammar)->complete_fsm.NumStates(), false); for (int i = 0; i < grammar_->ImplPtr()->complete_fsm.NumStates(); i++) { for (const auto& edge : grammar->ImplPtr()->complete_fsm.GetEdges(i)) { has_inward_edges_[edge.target] = true; } } // Get the reference graph. ref_graph_from_referee_to_referrer_ = RuleRefGraphFinder().Apply(*grammar); ref_graph_from_referrer_to_referee_ = std::vector>((*grammar)->NumRules()); for (int referee = 0; referee < static_cast(ref_graph_from_referee_to_referrer_.size()); ++referee) { for (int referer : ref_graph_from_referee_to_referrer_[referee]) { ref_graph_from_referrer_to_referee_[referer].push_back(referee); } } // Sort the edges. const auto& complete_fsm = grammar->ImplPtr()->complete_fsm; sorted_edges_.reserve(complete_fsm.NumStates()); for (int i = 0; i < complete_fsm.NumStates(); i++) { const auto& edges = complete_fsm.GetEdges(i); sorted_edges_.emplace_back(); sorted_edges_.back().reserve(edges.size()); for (const auto& edge : edges) { sorted_edges_.back().emplace_back(edge); } std::sort(sorted_edges_.back().begin(), sorted_edges_.back().end()); } // Disable non-fsms. for (size_t i = 0; i < grammar->ImplPtr()->per_rule_fsms.size(); i++) { if (!grammar->ImplPtr()->per_rule_fsms[i].has_value()) { visited_[i] = true; } } // Find the fsm which can be hashed: a terminal fsm, or a self-recursion fsm. auto current_operating_index = FindSimpleFsmCanBeHashed(); while (current_operating_index != -1) { visited_[current_operating_index] = true; grammar->ImplPtr()->per_rule_fsm_hashes[current_operating_index] = HashFsm(current_operating_index); // Remove the fsm from the reference graph. for (const auto& referer : ref_graph_from_referee_to_referrer_[current_operating_index]) { ref_graph_from_referrer_to_referee_[referer].erase(std::find_if( ref_graph_from_referrer_to_referee_[referer].begin(), ref_graph_from_referrer_to_referee_[referer].end(), [&](int32_t rule_id) { return rule_id == current_operating_index; } )); } // Find if there are more fsms can be hashed. current_operating_index = FindSimpleFsmCanBeHashed(); } // Try to hash the remaining fsms: they must contain something can't be hashed, like repetition. // We can do this: if the fsm's start state has no inward edges, and all the ref edges are hashed // except the edges at the start state, we can hash it. std::vector> partial_hashed_list; for (int i = 0; i < (*grammar)->NumRules(); i++) { if (grammar->ImplPtr()->per_rule_fsm_hashes[i].has_value()) { continue; } if (!grammar->ImplPtr()->per_rule_fsms[i].has_value()) { continue; } if (has_inward_edges_[grammar->ImplPtr()->per_rule_fsms[i]->GetFsm().GetStart()]) { continue; } const auto& [can_be_hashed, hash_value] = IsPartialHashable(i); if (can_be_hashed) { partial_hashed_list.emplace_back(i, hash_value); } } for (const auto& [rule_id, hash_value] : partial_hashed_list) { grammar->ImplPtr()->per_rule_fsm_hashes[rule_id] = hash_value; } } std::pair GrammarFSMHasherImpl::IsPartialHashable(int fsm_index) { uint64_t hash_result = 0; XGRAMMAR_DCHECK(fsm_index >= 0 && fsm_index < (*grammar_)->NumRules()) << "Invalid fsm index: " << fsm_index << " num_rules: " << (*grammar_)->NumRules(); XGRAMMAR_DCHECK(grammar_->ImplPtr()->per_rule_fsms[fsm_index].has_value()); const auto& fsm = grammar_->ImplPtr()->per_rule_fsms[fsm_index].value().GetFsm(); std::map original_state_id_to_new_id; original_state_id_to_new_id[fsm.GetStart()] = 0; std::queue bfs_queue; std::set> hash_and_target; bfs_queue.push(fsm.GetStart()); // Perform a bfs to hash all the edges. while (!bfs_queue.empty()) { int current_old_state_id = bfs_queue.front(); bool is_start = current_old_state_id == fsm.GetStart(); int current_new_state_id = original_state_id_to_new_id[current_old_state_id]; bfs_queue.pop(); // Check if the current state is an end state. if (fsm.IsEndState(current_old_state_id)) { hash_result = HashCombine( hash_result, current_new_state_id, kEndStateFlag, kEndStateFlag, current_new_state_id ); } else { hash_result = HashCombine( hash_result, current_new_state_id, kNotEndStateFlag, kNotEndStateFlag, current_new_state_id ); } // Hash the edges. // First, check the edges which are rule references (including repeat refs). // To keep consistent, we need to sort them with hashes. int32_t unhashed_rules_count = 0; auto hash_rule_like_edge = [&](int32_t ref_rule_id, int32_t target) { if (ref_rule_id == fsm_index) { hash_and_target.insert({kSelfRecursionFlag, target}); return true; } if (!grammar_->ImplPtr()->per_rule_fsm_hashes[ref_rule_id].has_value()) { if (!is_start) { return false; } else { unhashed_rules_count++; if (unhashed_rules_count > 1) { return false; } hash_and_target.insert({kUnKnownFlag, target}); } return true; } hash_and_target.insert({grammar_->ImplPtr()->per_rule_fsm_hashes[ref_rule_id].value(), target} ); return true; }; for (const auto& edge : sorted_edges_[current_old_state_id]) { if (edge.IsRuleRef()) { if (!hash_rule_like_edge(edge.GetRefRuleId(), edge.target)) { return {false, 0}; } } else if (edge.IsRepeatRef()) { auto info = grammar_->ImplPtr()->complete_fsm.GetRepeatEdgeInfo(edge.GetAuxIndex()); if (!hash_rule_like_edge(info.RuleId(), edge.target)) { return {false, 0}; } } } // Hash them. for (const auto& [hash, target] : hash_and_target) { if (original_state_id_to_new_id.find(target) == original_state_id_to_new_id.end()) { original_state_id_to_new_id[target] = static_cast(original_state_id_to_new_id.size()); bfs_queue.push(target); } int32_t target_new_id = original_state_id_to_new_id[target]; hash_result = HashCombine(hash_result, current_new_state_id, hash, target_new_id); } // Then, check the edges which are not rule/repeat references. for (const auto& edge : sorted_edges_[current_old_state_id]) { if (original_state_id_to_new_id.find(edge.target) == original_state_id_to_new_id.end()) { original_state_id_to_new_id[edge.target] = static_cast(original_state_id_to_new_id.size()); bfs_queue.push(edge.target); } int32_t target_new_id = original_state_id_to_new_id[edge.target]; if (edge.IsRuleRef() || edge.IsRepeatRef()) { continue; } hash_result = HashCombine( hash_result, current_new_state_id, static_cast(edge.min), static_cast(edge.max), target_new_id ); } } std::vector> new_id_mapping; new_id_mapping.reserve(original_state_id_to_new_id.size()); for (const auto& [original_state_id, new_state_id] : original_state_id_to_new_id) { new_id_mapping.emplace_back(original_state_id, new_state_id); } grammar_->ImplPtr()->per_rule_fsm_new_state_ids[fsm_index] = new_id_mapping; return {true, hash_result}; } uint64_t GrammarFSMHasherImpl::HashFsm(int fsm_index) { uint64_t hash_result = 0; XGRAMMAR_DCHECK(fsm_index >= 0 && fsm_index < (*grammar_)->NumRules()) << "Invalid fsm index: " << fsm_index << " num_rules: " << (*grammar_)->NumRules(); XGRAMMAR_DCHECK(grammar_->ImplPtr()->per_rule_fsms[fsm_index].has_value()); const auto& fsm = grammar_->ImplPtr()->per_rule_fsms[fsm_index].value().GetFsm(); std::map original_state_id_to_new_id; original_state_id_to_new_id[fsm.GetStart()] = 0; std::queue bfs_queue; std::set> hash_and_target; bfs_queue.push(fsm.GetStart()); // Perform a bfs to hash all the edges. while (!bfs_queue.empty()) { int current_old_state_id = bfs_queue.front(); int current_new_state_id = original_state_id_to_new_id[current_old_state_id]; bfs_queue.pop(); // Check if the current state is an end state. if (fsm.IsEndState(current_old_state_id)) { hash_result = HashCombine( hash_result, current_new_state_id, kEndStateFlag, kEndStateFlag, current_new_state_id ); } else { hash_result = HashCombine( hash_result, current_new_state_id, kNotEndStateFlag, kNotEndStateFlag, current_new_state_id ); } // Hash the edges. // First, check the edges which are rule references (including repeat refs). // To keep consistent, we need to sort them with hashes. for (const auto& edge : sorted_edges_[current_old_state_id]) { if (edge.IsRuleRef()) { int32_t ref_rule_id = edge.GetRefRuleId(); if (ref_rule_id == fsm_index) { hash_and_target.insert({kSelfRecursionFlag, edge.target}); } else { XGRAMMAR_CHECK(grammar_->ImplPtr()->per_rule_fsm_hashes[ref_rule_id].has_value()); hash_and_target.insert( {grammar_->ImplPtr()->per_rule_fsm_hashes[ref_rule_id].value(), edge.target} ); } } else if (edge.IsRepeatRef()) { auto info = grammar_->ImplPtr()->complete_fsm.GetRepeatEdgeInfo(edge.GetAuxIndex()); int32_t ref_rule_id = info.RuleId(); if (ref_rule_id == fsm_index) { uint64_t base_hash = kSelfRecursionFlag; uint64_t repeat_hash = HashCombine(base_hash, info.Lower(), info.Upper()); hash_and_target.insert({repeat_hash, edge.target}); } else { XGRAMMAR_CHECK(grammar_->ImplPtr()->per_rule_fsm_hashes[ref_rule_id].has_value()); uint64_t base_hash = grammar_->ImplPtr()->per_rule_fsm_hashes[ref_rule_id].value(); uint64_t repeat_hash = HashCombine(base_hash, info.Lower(), info.Upper()); hash_and_target.insert({static_cast(repeat_hash), edge.target}); } } } // Hash them. for (const auto& [hash, target] : hash_and_target) { if (original_state_id_to_new_id.find(target) == original_state_id_to_new_id.end()) { original_state_id_to_new_id[target] = static_cast(original_state_id_to_new_id.size()); bfs_queue.push(target); } int32_t target_new_id = original_state_id_to_new_id[target]; hash_result = HashCombine(hash_result, current_new_state_id, hash, target_new_id); } // Then, check the edges which are not rule/repeat references. for (const auto& edge : sorted_edges_[current_old_state_id]) { if (original_state_id_to_new_id.find(edge.target) == original_state_id_to_new_id.end()) { original_state_id_to_new_id[edge.target] = static_cast(original_state_id_to_new_id.size()); bfs_queue.push(edge.target); } int32_t target_new_id = original_state_id_to_new_id[edge.target]; if (edge.IsRuleRef() || edge.IsRepeatRef()) { continue; } hash_result = HashCombine( hash_result, current_new_state_id, static_cast(edge.min), static_cast(edge.max), target_new_id ); } } std::vector> new_id_mapping; new_id_mapping.reserve(original_state_id_to_new_id.size()); for (const auto& [original_state_id, new_state_id] : original_state_id_to_new_id) { new_id_mapping.emplace_back(original_state_id, new_state_id); } grammar_->ImplPtr()->per_rule_fsm_new_state_ids[fsm_index] = new_id_mapping; return hash_result; } std::optional GrammarFSMHasherImpl::HashSequence( const Grammar& grammar, int32_t sequence_id ) { using GrammarExprType = Grammar::Impl::GrammarExprType; if (sequence_id == -1) { return std::nullopt; } uint64_t hash_result = 0; const auto& sequence_expr = grammar->GetGrammarExpr(sequence_id); XGRAMMAR_DCHECK(sequence_expr.type == GrammarExprType::kSequence) << "GrammarExpr is not a sequence"; for (const auto& expr_id : sequence_expr) { const auto& expr = grammar->GetGrammarExpr(expr_id); hash_result = HashCombine(hash_result, static_cast(expr.type)); switch (expr.type) { case (GrammarExprType::kByteString): case (GrammarExprType::kCharacterClass): case (GrammarExprType::kCharacterClassStar): case (GrammarExprType::kEmptyStr): { for (const auto& element : expr) { hash_result = HashCombine(hash_result, element); } break; } case (GrammarExprType::kRuleRef): { if (grammar->per_rule_fsm_hashes[expr[0]].has_value()) { hash_result = HashCombine(hash_result, grammar->per_rule_fsm_hashes[expr[0]].value()); } else { return std::nullopt; } break; } case (GrammarExprType::kRepeat): { if (grammar->per_rule_fsm_hashes[expr[0]].has_value()) { hash_result = HashCombine(hash_result, grammar->per_rule_fsm_hashes[expr[0]].value()); } else { return std::nullopt; } hash_result = HashCombine(hash_result, expr[1]); hash_result = HashCombine(hash_result, expr[2]); break; } case (GrammarExprType::kSequence): case (GrammarExprType::kChoices): { return std::nullopt; } case (GrammarExprType::kTagDispatch): case (GrammarExprType::kTokenTagDispatch): { return std::nullopt; } case (GrammarExprType::kToken): case (GrammarExprType::kExcludeToken): { for (const auto& element : expr) { hash_result = HashCombine(hash_result, element); } break; } } } return hash_result; } class RuleLevelCache::Impl { public: using NodeKey = std::tuple< uint64_t /*The hash value of the FSM*/, int32_t /* The normalized node id*/, int32_t /*The number of states*/, int32_t /* The number of edges*/>; using NodeType = std::pair; explicit Impl(size_t max_cache_memory_size) : max_cache_memory_size_(max_cache_memory_size) {} std::optional GetCache( const uint64_t& fsm_hash, int32_t fsm_new_node_id, const int32_t& state_cnt, const int32_t edge_cnt ); bool AddCache( const uint64_t& fsm_hash, int32_t fsm_new_node_id, const int32_t& state_cnt, const int32_t edge_cnt, const AdaptiveTokenMask& token_mask ); bool AddCache( const uint64_t& fsm_hash, int32_t fsm_new_node_id, const int32_t& state_cnt, const int32_t edge_cnt, AdaptiveTokenMask&& token_mask ); void ClearCache(); friend size_t MemorySize(const Impl* impl) { return impl->current_cache_memory_size_; } size_t GetMaxSize() const { return max_cache_memory_size_; } private: // The cache map: fsm_hash -> fsm_new_node_id -> AdaptiveTokenMask std::mutex mutex_; const size_t max_cache_memory_size_; int64_t current_cache_memory_size_ = 0; List cache_list_; std::unordered_map cache_; }; std::optional RuleLevelCache::GetCache( const uint64_t& fsm_hash, int32_t fsm_new_node_id, const int32_t& state_cnt, const int32_t edge_cnt ) { return pimpl_->GetCache(fsm_hash, fsm_new_node_id, state_cnt, edge_cnt); } bool RuleLevelCache::AddCache( const uint64_t& fsm_hash, int32_t fsm_new_node_id, const int32_t& state_cnt, const int32_t edge_cnt, const AdaptiveTokenMask& token_mask ) { return pimpl_->AddCache(fsm_hash, fsm_new_node_id, state_cnt, edge_cnt, token_mask); } bool RuleLevelCache::AddCache( const uint64_t& fsm_hash, int32_t fsm_new_node_id, const int32_t& state_cnt, const int32_t edge_cnt, AdaptiveTokenMask&& token_mask ) { return pimpl_->AddCache(fsm_hash, fsm_new_node_id, state_cnt, edge_cnt, std::move(token_mask)); } void RuleLevelCache::ClearCache() { pimpl_->ClearCache(); } size_t RuleLevelCache::GetMaxSize() const { return pimpl_->GetMaxSize(); } std::optional RuleLevelCache::Impl::GetCache( const uint64_t& fsm_hash, int32_t fsm_new_node_id, const int32_t& state_cnt, const int32_t edge_cnt ) { // Find in the cache. std::lock_guard lock(mutex_); NodeKey key = std::make_tuple(fsm_hash, fsm_new_node_id, state_cnt, edge_cnt); auto it = cache_.find(key); if (it == cache_.end()) { return std::nullopt; } // Move the node to the back of the list. cache_list_.MoveBack(it->second); return List::iterator(it->second, cache_list_)->second; } bool RuleLevelCache::Impl::AddCache( const uint64_t& fsm_hash, int32_t fsm_new_node_id, const int32_t& state_cnt, const int32_t edge_cnt, const AdaptiveTokenMask& token_mask ) { return AddCache(fsm_hash, fsm_new_node_id, state_cnt, edge_cnt, AdaptiveTokenMask(token_mask)); } bool RuleLevelCache::Impl::AddCache( const uint64_t& fsm_hash, int32_t fsm_new_node_id, const int32_t& state_cnt, const int32_t edge_cnt, AdaptiveTokenMask&& token_mask ) { // Check if we can add to the cache. std::lock_guard lock(mutex_); NodeKey key = std::make_tuple(fsm_hash, fsm_new_node_id, state_cnt, edge_cnt); if (max_cache_memory_size_ != kUnlimitedSize && MemorySize(token_mask) > max_cache_memory_size_) { // The token mask is too large to be cached. return false; } if (cache_.find(key) != cache_.end()) { // Already exists. return false; } // Evict old entries if needed. if (max_cache_memory_size_ != kUnlimitedSize) { size_t new_item_size = MemorySize(token_mask); while ((current_cache_memory_size_) > static_cast(max_cache_memory_size_ - new_item_size)) { auto oldest_it = cache_list_.begin(); if (oldest_it == cache_list_.end()) { // This should not happen if the size of the new item is smaller than // max_cache_memory_size_, but this is a safeguard. break; } current_cache_memory_size_ -= MemorySize(oldest_it->second); cache_.erase(oldest_it->first); cache_list_.Erase(oldest_it); } } // Add to the cache. auto new_it = cache_list_.PushBack(NodeType(key, std::move(token_mask))); current_cache_memory_size_ += MemorySize(new_it->second); cache_[key] = new_it.Index(); return true; } RuleLevelCache::RuleLevelCache(size_t max_cache_memory_size) : pimpl_(std::make_shared(max_cache_memory_size)) {} void RuleLevelCache::Impl::ClearCache() { std::lock_guard lock(mutex_); cache_list_.Clear(); cache_.clear(); current_cache_memory_size_ = 0; } size_t MemorySize(const RuleLevelCache& manager) { return MemorySize(manager.ImplPtr()); } /*************************** Forward grammar constructors to their impl ***************************/ Grammar GrammarUnionFunctor::Apply(const std::vector& grammars) { return GrammarUnionFunctorImpl().Apply(grammars); } Grammar GrammarConcatFunctor::Apply(const std::vector& grammars) { return GrammarConcatFunctorImpl().Apply(grammars); } int32_t SubGrammarAdder::Apply(GrammarBuilder* builder, const Grammar& sub_grammar) { return SubGrammarAdderImpl().ApplyWithBuilder(builder, sub_grammar); } /*************************** Forward grammar Normalizers to their impl ***************************/ Grammar GrammarNormalizer::Apply(const Grammar& grammar) { return GrammarNormalizerImpl().Apply(grammar); } Grammar StructureNormalizer::Apply(const Grammar& grammar) { return StructureNormalizerImpl().Apply(grammar); } /*************************** Forward grammar optimizers to their impl ***************************/ void GrammarFSMBuilder::Apply(Grammar* grammar) { GrammarFSMBuilderImpl().Apply(grammar); } void RepetitionNormalizer::Apply(Grammar* grammar) { RepetitionNormalizerImpl().Apply(grammar); } void GrammarFSMHasher::Apply(Grammar* grammar) { GrammarFSMHasherImpl().Apply(grammar); } std::optional GrammarFSMHasher::HashSequence( const Grammar& grammar, int32_t sequence_id ) { return GrammarFSMHasherImpl().HashSequence(grammar, sequence_id); } FSMWithStartEnd GrammarFSMBuilder::RuleRef(const GrammarExpr& expr) { return GrammarFSMBuilderImpl::RuleRef(expr); } FSMWithStartEnd GrammarFSMBuilder::CharacterClass(const GrammarExpr& expr) { return GrammarFSMBuilderImpl::CharacterClass(expr); } FSMWithStartEnd GrammarFSMBuilder::ByteString(const GrammarExpr& expr) { return GrammarFSMBuilderImpl::ByteString(expr); } FSMWithStartEnd GrammarFSMBuilder::Token(const GrammarExpr& expr) { return GrammarFSMBuilderImpl::Token(expr); } FSMWithStartEnd GrammarFSMBuilder::ExcludeToken(const GrammarExpr& expr) { return GrammarFSMBuilderImpl::ExcludeToken(expr); } std::optional GrammarFSMBuilder::TokenTagDispatch( const Grammar::Impl::TokenTagDispatch& ttd ) { return GrammarFSMBuilderImpl::TokenTagDispatch(ttd); } std::optional GrammarFSMBuilder::Sequence( const GrammarExpr& expr, const Grammar& grammar ) { return GrammarFSMBuilderImpl::Sequence(expr, grammar); } std::optional GrammarFSMBuilder::Choices( const GrammarExpr& expr, const Grammar& grammar ) { return GrammarFSMBuilderImpl::Choices(expr, grammar); } std::optional GrammarFSMBuilder::TagDispatch( const Grammar::Impl::TagDispatch& tag_dispatch ) { return GrammarFSMBuilderImpl::TagDispatch(tag_dispatch); } std::vector AllowEmptyRuleAnalyzer::Apply(const Grammar& grammar) { return AllowEmptyRuleAnalyzerImpl().Apply(grammar); } Grammar RuleInliner::Apply(const Grammar& grammar) { return RuleInlinerImpl().Apply(grammar); } Grammar DeadCodeEliminator::Apply(const Grammar& grammar) { return DeadCodeEliminatorImpl().Apply(grammar); } Grammar LookaheadAssertionAnalyzer::Apply(const Grammar& grammar) { return LookaheadAssertionAnalyzerImpl().Apply(grammar); } Grammar RepetitionRangeExpander::Apply(const Grammar& grammar) { return RepetitionRangeExpanderImpl().Apply(grammar); } Grammar GrammarOptimizer::Apply(const Grammar& grammar) { return GrammarOptimizerImpl::Apply(grammar); } Grammar ByteStringFuser::Apply(const Grammar& grammar) { return ByteStringFuserImpl().Apply(grammar); } Grammar RootRuleRenamer::Apply(const Grammar& grammar) { return RootRuleRenamerImpl().Apply(grammar); } } // namespace xgrammar xgrammar-0.2.3/cpp/grammar_functor.h000066400000000000000000000347211521764210300174630ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar_functor.h * \brief The header for the simplification of the BNF AST. */ #ifndef XGRAMMAR_GRAMMAR_FUNCTOR_H_ #define XGRAMMAR_GRAMMAR_FUNCTOR_H_ #include #include #include #include #include "compiled_grammar_impl.h" #include "grammar_builder.h" #include "grammar_impl.h" #include "xgrammar/grammar.h" namespace xgrammar { /*! * \brief Base class for visitors and mutators of the BNF grammar. * \tparam T The type of the return value of visitor functions. Typical values: * - int32_t: the id of the new grammar_expr * - void: no return value * \tparam ReturnType The type of the return value of the transform function Apply(). Typical values * are void (for visitor) and Grammar (for mutator). */ template class GrammarFunctor { public: /*! * \brief Constructor. * \param grammar The grammar to visit or mutate. */ explicit GrammarFunctor() {} /*! * \brief Apply the transformation to the grammar, or visit the grammar. * \return The transformed grammar, or the visiting result, or void. */ virtual ReturnType Apply(const Grammar& grammar) { // The initializer MUST be called at first when overriding the Apply() function. InitGrammar(grammar); if constexpr (std::is_same::value) { for (int i = 0; i < static_cast(base_grammar_->NumRules()); ++i) { auto rule = base_grammar_->GetRule(i); cur_rule_name_ = rule.name; VisitExpr(rule.body_expr_id); VisitLookaheadAssertion(rule.lookahead_assertion_id); } return ReturnType(); } else if constexpr (std::is_same::value && std::is_same::value) { InitBuilder(); // First add empty rules to ensure the new rule ids the same as the old ones, then update // the rule bodies for (int i = 0; i < static_cast(base_grammar_->NumRules()); ++i) { builder_->AddEmptyRule(base_grammar_->GetRule(i).name); } for (int i = 0; i < static_cast(base_grammar_->NumRules()); ++i) { auto rule = base_grammar_->GetRule(i); cur_rule_name_ = rule.name; auto new_body_expr_id = VisitExpr(rule.body_expr_id); builder_->UpdateRuleBody(i, new_body_expr_id); // Handle lookahead assertion builder_->UpdateLookaheadAssertion(i, VisitLookaheadAssertion(rule.lookahead_assertion_id)); } return builder_->Get(base_grammar_->GetRootRule().name); } else { return ReturnType(); } } /*! \brief Virtual destructor. */ virtual ~GrammarFunctor() = default; protected: using Rule = Grammar::Impl::Rule; using GrammarExpr = Grammar::Impl::GrammarExpr; using GrammarExprType = Grammar::Impl::GrammarExprType; /*! \brief Initialize the functor. Should be called at the beginning of Apply(). */ virtual void InitGrammar() {} virtual void InitGrammar(const Grammar& grammar) { base_grammar_ = grammar; } virtual void InitBuilder() { owned_builder_ = GrammarBuilder(); builder_ = &owned_builder_; } virtual void InitBuilder(const Grammar& grammar) { owned_builder_ = GrammarBuilder(grammar); builder_ = &owned_builder_; } virtual void InitBuilder(GrammarBuilder* builder) { builder_ = builder; } /*! \brief Visit a lookahead assertion expr referred by id. */ virtual T VisitLookaheadAssertion(int32_t lookahead_assertion_id) { if (lookahead_assertion_id == -1) { if constexpr (std::is_same::value) { return -1; } else { return T(); } } return VisitExpr(lookahead_assertion_id); } /*! \brief Visit a GrammarExpr by id. */ virtual T VisitExpr(int32_t old_grammar_expr_id) { return VisitExpr(base_grammar_->GetGrammarExpr(old_grammar_expr_id)); } /*! \brief Visit a GrammarExpr. Dispatch to the corresponding Visit function. */ virtual T VisitExpr(const GrammarExpr& grammar_expr) { switch (grammar_expr.type) { case GrammarExprType::kSequence: return VisitSequence(grammar_expr); case GrammarExprType::kChoices: return VisitChoices(grammar_expr); case GrammarExprType::kEmptyStr: return VisitEmptyStr(grammar_expr); case GrammarExprType::kByteString: return VisitByteString(grammar_expr); case GrammarExprType::kCharacterClass: return VisitCharacterClass(grammar_expr); case GrammarExprType::kCharacterClassStar: return VisitCharacterClassStar(grammar_expr); case GrammarExprType::kRuleRef: return VisitRuleRef(grammar_expr); case GrammarExprType::kTagDispatch: return VisitTagDispatch(grammar_expr); case GrammarExprType::kRepeat: return VisitRepeat(grammar_expr); case GrammarExprType::kToken: return VisitToken(grammar_expr); case GrammarExprType::kExcludeToken: return VisitExcludeToken(grammar_expr); case GrammarExprType::kTokenTagDispatch: return VisitTokenTagDispatch(grammar_expr); default: XGRAMMAR_LOG(FATAL) << "Unexpected sequence type: " << static_cast(grammar_expr.type); XGRAMMAR_UNREACHABLE(); } } /*! \brief Visit a choices GrammarExpr. */ virtual T VisitChoices(const GrammarExpr& grammar_expr) { if constexpr (std::is_same::value) { for (auto i : grammar_expr) { VisitExpr(i); } } else if constexpr (std::is_same::value) { std::vector choice_ids; for (int32_t i : grammar_expr) { choice_ids.push_back(VisitExpr(i)); } return builder_->AddChoices(choice_ids); } else { return T(); } } /*! \brief Visit a sequence GrammarExpr. */ virtual T VisitSequence(const GrammarExpr& grammar_expr) { if constexpr (std::is_same::value) { for (auto i : grammar_expr) { VisitExpr(i); } } else if constexpr (std::is_same::value) { std::vector sequence_ids; for (int32_t i : grammar_expr) { sequence_ids.push_back(VisitExpr(i)); } return builder_->AddSequence(sequence_ids); } else { return T(); } } virtual T VisitTagDispatch(const GrammarExpr& grammar_expr) { if constexpr (std::is_same::value) { return; } else if constexpr (std::is_same::value) { Grammar::Impl::TagDispatch tag_dispatch = base_grammar_->GetTagDispatch(grammar_expr); return builder_->AddTagDispatch(tag_dispatch); } else { return T(); } } /*! \brief Visit an element GrammarExpr, including empty string, character class, and rule ref. */ virtual T VisitElement(const GrammarExpr& grammar_expr) { if constexpr (std::is_same::value) { return; } else if constexpr (std::is_same::value) { return builder_->AddGrammarExpr(grammar_expr); } else { return T(); } } /*! \brief Visit an empty string GrammarExpr. */ virtual T VisitEmptyStr(const GrammarExpr& grammar_expr) { return VisitElement(grammar_expr); } /*! \brief Visit a character class GrammarExpr. */ virtual T VisitByteString(const GrammarExpr& grammar_expr) { return VisitElement(grammar_expr); } /*! \brief Visit a character class GrammarExpr. */ virtual T VisitCharacterClass(const GrammarExpr& grammar_expr) { return VisitElement(grammar_expr); } /*! \brief Visit a star quantifier GrammarExpr. */ virtual T VisitCharacterClassStar(const GrammarExpr& grammar_expr) { return VisitElement(grammar_expr); } /*! \brief Visit a rule reference GrammarExpr. */ virtual T VisitRuleRef(const GrammarExpr& grammar_expr) { return VisitElement(grammar_expr); } /*! \brief Visit a repeat GrammarExpr. */ virtual T VisitRepeat(const GrammarExpr& grammar_expr) { return VisitElement(grammar_expr); } virtual T VisitToken(const GrammarExpr& grammar_expr) { return VisitElement(grammar_expr); } virtual T VisitExcludeToken(const GrammarExpr& grammar_expr) { return VisitElement(grammar_expr); } virtual T VisitTokenTagDispatch(const GrammarExpr& grammar_expr) { return VisitElement(grammar_expr); } /*! \brief The grammar to visit or mutate. */ Grammar base_grammar_{NullObj{}}; /*! * \brief The builder to build the new grammar. It is empty when the mutator is constructed, and * can be used to build a new grammar in subclasses. */ GrammarBuilder* builder_ = nullptr; GrammarBuilder owned_builder_; /*! \brief The name of the current rule being visited. */ std::string cur_rule_name_; }; /*! * \brief Visitor of Grammar. * \tparam ReturnType The return type of the Apply() function. Denotes the collected information. */ template using GrammarVisitor = GrammarFunctor; /*! * \brief Mutator of Grammar. The Apply() function returns the updated grammar. */ using GrammarMutator = GrammarFunctor; /****** All below methods are implemented as functor to hide the implementation ******/ /*************************** Grammar Constructor ***************************/ /*! * \brief Find the union of multiple grammars as a new grammar. */ class GrammarUnionFunctor { public: static Grammar Apply(const std::vector& grammars); }; /*! * \brief Find the concatenation of multiple grammars as a new grammar. */ class GrammarConcatFunctor { public: static Grammar Apply(const std::vector& grammars); }; /*! * \brief Add a sub grammar to the current builder. The return value * of Apply is the new rule id of the sub grammar's root rule. */ class SubGrammarAdder { public: static int32_t Apply(GrammarBuilder* builder, const Grammar& sub_grammar); }; /*************************** Grammar Normalizer ***************************/ /*! * \brief Normalize a Grammar: expand the nested rules, combine consequent sequences and strings, * etc. */ class GrammarNormalizer { public: static Grammar Apply(const Grammar& grammar); }; /*! * \brief Normalize the structure of the grammar. It will ensure each rule is a choices of * sequences of elements, or a tag dispatch. The expanded context will be a sequence of elements. */ class StructureNormalizer { public: static Grammar Apply(const Grammar& grammar); }; /*************************** Grammar Optimizer ***************************/ /*! * \brief Fuse the byte string elements in the grammar. */ class ByteStringFuser { public: static Grammar Apply(const Grammar& grammar); }; /*! * \brief Analyze the grammar to find the rules that are allowed to be empty. */ class AllowEmptyRuleAnalyzer { public: static std::vector Apply(const Grammar& grammar); }; /*! * \brief Inline the rule references in the grammar. */ class RuleInliner { public: static Grammar Apply(const Grammar& grammar); }; /*! * \brief Eliminate the not referenced rules in the grammar. */ class DeadCodeEliminator { public: static Grammar Apply(const Grammar& grammar); }; /*! * \brief Analyze and add lookahead assertions in the grammar. */ class LookaheadAssertionAnalyzer { public: static Grammar Apply(const Grammar& grammar); }; /*! * \brief Build the FSMs of the grammar. */ class GrammarFSMBuilder { using GrammarExpr = Grammar::Impl::GrammarExpr; public: static void Apply(Grammar* grammar); static FSMWithStartEnd RuleRef(const GrammarExpr& expr); static FSMWithStartEnd CharacterClass(const GrammarExpr& expr); static FSMWithStartEnd ByteString(const GrammarExpr& expr); static FSMWithStartEnd Token(const GrammarExpr& expr); static FSMWithStartEnd ExcludeToken(const GrammarExpr& expr); static std::optional TokenTagDispatch( const Grammar::Impl::TokenTagDispatch& token_tag_dispatch ); static std::optional Sequence(const GrammarExpr& expr, const Grammar& grammar); static std::optional Choices(const GrammarExpr& expr, const Grammar& grammar); static std::optional TagDispatch(const Grammar::Impl::TagDispatch& tag_dispatch); }; /*! * \brief Normalize the repetition expression. If the context of * repetition expression is nullable, then the repetition range will be * normalized from {m, n} to {0, n} to reduce uncertainty. */ class RepetitionNormalizer { public: static void Apply(Grammar* grammar); }; /*! * \brief Expand kRepeat grammar expressions using HandleRepetitionRange logic. * Transforms repetition structures into explicit sequences and choices. */ class RepetitionRangeExpander { public: static Grammar Apply(const Grammar& grammar); }; /*! * \brief Optimize the grammar when compiling. * \note No matter whether the grammar is optimized, grammar optimizer will * return a new grammar. The following optimization will be applied: * 1. Byte fuser. * 2. Rule inliner. * 3. Dead code eliminator. * 4. Lookahead assertion analyzer. * 5. Allow-empty rule analyzer. * 6. Repetition normalizer. * 7. FSM builder. */ class GrammarOptimizer { public: static Grammar Apply(const Grammar& grammar); }; /*! * \brief Rename the root rule of the grammar to "root". */ class RootRuleRenamer { public: static Grammar Apply(const Grammar& grammar); }; /*! * \brief Hash the fsms in the grammar, * and get the new state ids of each fsm's states. */ class GrammarFSMHasher { public: static void Apply(Grammar* grammar); static std::optional HashSequence(const Grammar& grammar, int32_t sequence_id); }; /*! * \brief Store the crossing cache for different grammars. * \param max_cache_size The maximum size of the cache numbers. * \details LRU algorithm is implemented. */ class RuleLevelCache { public: static const size_t kUnlimitedSize = static_cast(-1); std::optional GetCache( const uint64_t& fsm_hash, int32_t fsm_new_node_id, const int32_t& state_cnt, const int32_t edge_cnt ); bool AddCache( const uint64_t& fsm_hash, int32_t fsm_new_node_id, const int32_t& state_cnt, const int32_t edge_cnt, const AdaptiveTokenMask& token_mask ); bool AddCache( const uint64_t& fsm_hash, int32_t fsm_new_node_id, const int32_t& state_cnt, const int32_t edge_cnt, AdaptiveTokenMask&& token_mask ); RuleLevelCache(size_t max_cache_memory_size = kUnlimitedSize); void ClearCache(); size_t GetMaxSize() const; friend size_t MemorySize(const RuleLevelCache& manager); XGRAMMAR_DEFINE_PIMPL_METHODS(RuleLevelCache); }; } // namespace xgrammar #endif // XGRAMMAR_GRAMMAR_FUNCTOR_H_ xgrammar-0.2.3/cpp/grammar_impl.h000066400000000000000000000317761521764210300167530ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar.h * \brief The header for the support of grammar-guided generation. */ #ifndef XGRAMMAR_GRAMMAR_IMPL_H_ #define XGRAMMAR_GRAMMAR_IMPL_H_ #include #include #include #include #include "fsm.h" #include "support/logging.h" #include "support/reflection.h" #include "xgrammar/grammar.h" namespace xgrammar { /*! * \brief This class stores the abstract syntax tree (AST) of the Backus-Naur Form (BNF) grammar. * The BNF definition here is standard BNF, and the characters are represented using regex-style * character classes (e.g. [a-z], [^a-z]). * * \details * ### Rules * The BNF grammar AST consists of a set of rules. Each rule contains a name and a definition, and * corresponds to a production in the grammar. The definition of a rule is a GrammarExpr. Each rule * has a rule_id for reference. * * ### GrammarExprs * GrammarExpr is the definition of a rule or part of the definition of a rule. It can contain * elements, empty string, reference to other GrammarExprs, or reference to other rules. Each * GrammarExpr corresponds to a grammar_expr_id for reference. * * For example, in the following rule: rule ::= ("a" "b") | "c" * ("a" "b"), "c", ("a" "b") | "c" are all GrammarExprs. * * #### Types of GrammarExprs * Every GrammarExpr is represented by a type as well as a variable-length array containing its * data. GrammarExpr has several types: * - Byte string: a string of bytes (0~255). Supports UTF-8 strings. * - Character class: a range of characters (each character is a unicode codepoint), e.g. [a-z], * [ac-z]. Can be negated: [^a-z], [^ac-z]. Now only ascii chars is allowed in [], but this * expression can accept/reject unicode chars. * - Character class star: a star quantifier of a character class. e.g. [a-z]*, [^a-z]*. * - EmptyStr: an empty string, i.e. "" * - Rule reference: a reference to another rule * - Sequence: a sequence of grammar_exprs, e.g. ("a" "b"). These grammar_exprs are concatenated * together. * - Choices: a choice of grammar_exprs, e.g. ("a" "b") | "c". Each grammar_expr can be matched. * * #### Storage of GrammarExprs * Each type of GrammarExpr has a different data format. For the format of each type of GrammarExpr, * see docs in Grammar::Impl::GrammarExprType. * * We store all GrammarExprs in csr_matrix style. That is, they are stored consecutively in one * vector (data vector) and the starting position of each GrammarExpr is recorded in the indptr * vector. * * \remark The character class star GrammarExpr is for the special support for elements like [a-z]* * in the grammar. We add it to make the matching more efficient, as we can avoid recursion into * rules when matching a sequence of characters. It should be used like: * rule1 ::= ((element1 element2 rule2 ...) | ...) * rule2 ::= character_class_star_grammar_expr(id_of_a_character_class_grammar_expr) */ class Grammar::Impl { public: /*! \brief A rule with name. */ struct Rule { /*! \brief The name of the rule. */ std::string name; /*! \brief The GrammarExpr id of the body of the rule. */ int32_t body_expr_id; /*! \brief The id of the associated lookahead assertion expr. For now it must be a id of a * sequence GrammarExpr. -1 if not exists. */ int32_t lookahead_assertion_id = -1; /*! \brief Whether the lookahead assertion is exact. */ bool is_exact_lookahead = false; }; /*! \brief Get the number of rules. */ int32_t NumRules() const { return rules_.size(); } /*! \brief Get the rule with the given id. */ const Rule& GetRule(int32_t rule_id) const { XGRAMMAR_DCHECK(rule_id >= 0 && rule_id < static_cast(rules_.size())) << "rule_id " << rule_id << " is out of bound"; return rules_[rule_id]; } Rule& GetRule(int32_t rule_id) { XGRAMMAR_DCHECK(rule_id >= 0 && rule_id < static_cast(rules_.size())) << "rule_id " << rule_id << " is out of bound"; return rules_[rule_id]; } /*! \brief Get the root rule id of the grammar. */ int32_t GetRootRuleId() const { return root_rule_id_; } /*! \brief Get the root rule of the grammar. */ const Rule& GetRootRule() const { XGRAMMAR_DCHECK(root_rule_id_ >= 0 && root_rule_id_ < static_cast(rules_.size())) << "root_rule_id " << root_rule_id_ << " is out of bound"; return rules_[root_rule_id_]; } /*! \brief The type of the grammar expr. */ enum class GrammarExprType : int32_t { // data format: [byte0, byte1, ...] kByteString, // data format: [is_negative, lower0, upper0, lower1, upper1, ...] kCharacterClass, kCharacterClassStar, // data format: [] kEmptyStr, // data format: [rule_id] kRuleRef, // data format: [grammar_expr_id0, grammar_expr_id1, ...] kSequence, // data format: [grammar_expr_id0, grammar_expr_id1, ...] kChoices, // data format: [tag_expr0, rule_id0, tag_expr1, rule_id1, ..., loop_after_dispatch, // excluded_str_expr_id] kTagDispatch, // data format: [rule_id, min_repeat_count, max_repeat_count] kRepeat, // data format: [token_id_0, token_id_1, ...] kToken, // data format: [token_id_0, token_id_1, ...] kExcludeToken, // data format: [trigger_cnt, (token_id, rule_id) × N, // loop_after_dispatch, // exclude_cnt, token_id × M] kTokenTagDispatch, }; /*! \brief The object representing a grammar expr. */ struct GrammarExpr { /*! \brief The type of the grammar expr. */ GrammarExprType type; /*! \brief The data of the GrammarExpr. A variable-length array. */ const int32_t* data; /*! \brief The length of the data array. */ int32_t data_len; int32_t size() const { return data_len; } /*! \brief Get the i-th element of the data array. */ const int32_t& operator[](int i) const { XGRAMMAR_DCHECK(i >= 0 && i < static_cast(data_len)) << "Index " << i << " is out of bound"; return data[i]; } const int32_t* begin() const { return data; } const int32_t* end() const { return data + data_len; } void SetData(int index, int value) { const_cast(data)[index] = value; } }; /*! \brief Get the number of grammar_exprs. */ int32_t NumGrammarExprs() const { return grammar_expr_indptr_.size(); } /*! \brief Get the grammar_expr with the given id. */ GrammarExpr GetGrammarExpr(int32_t grammar_expr_id) const { XGRAMMAR_DCHECK( grammar_expr_id >= 0 && grammar_expr_id < static_cast(grammar_expr_indptr_.size()) ) << "grammar_expr_id " << grammar_expr_id << " is out of bound"; int start_index = grammar_expr_indptr_[grammar_expr_id]; auto start_ptr = grammar_expr_data_.data() + start_index; auto type = static_cast(start_ptr[0]); auto data_ptr = start_ptr + 2; auto data_len = start_ptr[1]; return {type, data_ptr, data_len}; } /******************* GrammarExpr Getters *******************/ /*! \brief Get the string of the byte string grammar expr. */ std::string GetByteString(const GrammarExpr& grammar_expr) const { std::string str; str.reserve(grammar_expr.size()); for (int i = 0; i < grammar_expr.size(); ++i) { str.push_back(static_cast(static_cast(grammar_expr[i]))); } return str; } /*! \brief Get the string of the byte string grammar expr. */ std::string GetByteString(int32_t grammar_expr_id) const { return GetByteString(GetGrammarExpr(grammar_expr_id)); } /*! \brief The object representing a tag dispatch. */ struct TagDispatch { /*! \brief The tag and rule id pairs. */ std::vector> tag_rule_pairs; /*! \brief If true, the tag dispatch will loop after dispatching. */ bool loop_after_dispatch; /*! \brief The strings that are excluded by the tag dispatch. */ std::vector excludes; static const int kTagDispatchExtraParameter = 2; }; /*! \brief Get the tag dispatch from the grammar expr. */ TagDispatch GetTagDispatch(const GrammarExpr& grammar_expr) { XGRAMMAR_DCHECK(grammar_expr.type == GrammarExprType::kTagDispatch) << "GrammarExpr is not a tag dispatch"; TagDispatch result; XGRAMMAR_DCHECK(grammar_expr.size() >= TagDispatch::kTagDispatchExtraParameter); result.tag_rule_pairs.reserve( (grammar_expr.size() - TagDispatch::kTagDispatchExtraParameter) / 2 ); for (int i = 0; i < grammar_expr.size() - TagDispatch::kTagDispatchExtraParameter; i += 2) { auto tag_expr_id = grammar_expr[i]; auto rule_id = grammar_expr[i + 1]; result.tag_rule_pairs.push_back({GetByteString(tag_expr_id), rule_id}); } result.loop_after_dispatch = static_cast( grammar_expr[grammar_expr.size() - TagDispatch::kTagDispatchExtraParameter] ); auto exclude_str_expr = GetGrammarExpr( grammar_expr[grammar_expr.size() - TagDispatch::kTagDispatchExtraParameter + 1] ); XGRAMMAR_DCHECK(exclude_str_expr.type == GrammarExprType::kChoices); result.excludes.reserve(exclude_str_expr.size()); for (int j = 0; j < exclude_str_expr.size(); j++) { result.excludes.push_back(GetByteString(exclude_str_expr[j])); } return result; } /*! \brief Get the tag dispatch from the grammar expr with the given id. */ TagDispatch GetTagDispatch(int32_t grammar_expr_id) { return GetTagDispatch(GetGrammarExpr(grammar_expr_id)); } /*! \brief The object representing a token tag dispatch. */ struct TokenTagDispatch { std::vector> trigger_rule_pairs; // token_id → rule_id bool loop_after_dispatch; std::vector excludes; }; /*! \brief Decode a kTokenTagDispatch expr into the TokenTagDispatch struct. */ TokenTagDispatch GetTokenTagDispatch(const GrammarExpr& grammar_expr) { XGRAMMAR_DCHECK(grammar_expr.type == GrammarExprType::kTokenTagDispatch); TokenTagDispatch result; int pos = 0; int32_t trigger_count = grammar_expr[pos++]; for (int i = 0; i < trigger_count; ++i) { auto token_id = grammar_expr[pos++]; auto rule_id = grammar_expr[pos++]; result.trigger_rule_pairs.push_back({token_id, rule_id}); } result.loop_after_dispatch = static_cast(grammar_expr[pos++]); int32_t exclude_count = grammar_expr[pos++]; for (int i = 0; i < exclude_count; ++i) { result.excludes.push_back(grammar_expr[pos++]); } XGRAMMAR_DCHECK(pos == grammar_expr.size()); return result; } /*! \brief Get the token tag dispatch from the grammar expr with the given id. */ TokenTagDispatch GetTokenTagDispatch(int32_t grammar_expr_id) { return GetTokenTagDispatch(GetGrammarExpr(grammar_expr_id)); } private: /*! \brief The rules of the grammar. rule_id corresponds the index of this vector. */ std::vector rules_; /*! \brief The data of all grammar_exprs. */ std::vector grammar_expr_data_; /*! \brief The start index of every grammar_expr in grammar_expr_data_. grammar_expr_id is the * index to the elements in this vector. */ std::vector grammar_expr_indptr_; /*! \brief The id of the root rule. */ int32_t root_rule_id_ = -1; public: /******************* Aux information for matching *******************/ /*! \brief The complete FSM for the grammar. It contains the FSMs for all rules. */ CompactFSM complete_fsm{NullObj{}}; /*! * \brief The FSM for each rule. * \details The FSM will be used in matching if it exists. If it does not exist (std::nullopt), * the rule will be used in matching, and the rule's body must be a kChoices expr. */ std::vector> per_rule_fsms; /*! * \brief The hash value for each rule's FSM. */ std::vector> per_rule_fsm_hashes; /*! * \brief The new state ids of each FSM's states. */ std::vector>> per_rule_fsm_new_state_ids; /*! \brief The ids of the rules that are allowed to be empty. */ std::vector allow_empty_rule_ids; /*! \brief Whether the grammar is optimized. */ bool optimized = false; friend class GrammarBuilder; friend class GrammarCompiler; friend std::size_t MemorySize(const Impl& impl); friend struct member_trait; }; XGRAMMAR_MEMBER_ARRAY( Grammar::Impl::Rule, &Grammar::Impl::Rule::name, &Grammar::Impl::Rule::body_expr_id, &Grammar::Impl::Rule::lookahead_assertion_id, &Grammar::Impl::Rule::is_exact_lookahead ); XGRAMMAR_MEMBER_TABLE( Grammar::Impl, "rules", &Grammar::Impl::rules_, "grammar_expr_data", &Grammar::Impl::grammar_expr_indptr_, "grammar_expr_indptr", &Grammar::Impl::grammar_expr_data_, "root_rule_id", &Grammar::Impl::root_rule_id_, "complete_fsm", &Grammar::Impl::complete_fsm, "per_rule_fsms", &Grammar::Impl::per_rule_fsms, "allow_empty_rule_ids", &Grammar::Impl::allow_empty_rule_ids, "optimized", &Grammar::Impl::optimized ); } // namespace xgrammar #endif // XGRAMMAR_GRAMMAR_IMPL_H_ xgrammar-0.2.3/cpp/grammar_matcher.cc000066400000000000000000001434101521764210300175600ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar_matcher.cc * \brief This source file implement the matcher class, especially the logic related to LLM tokens, * like accepting tokens, leveraging the token mask cache to generate the mask, etc. matcher_base.cc * implements the basic matching algorithm from strings to grammar. */ #include #include #include #include #include #include #include #include #include #include #include "compiled_grammar_impl.h" #include "earley_parser.h" #include "grammar_impl.h" #include "support/dynamic_bitset.h" #include "support/encoding.h" #include "support/int_set.h" #include "support/logging.h" #include "support/thread_pool.h" #include "testing.h" namespace xgrammar { /******************* Tool functions for token mask *******************/ using GrammarExprType = Grammar::Impl::GrammarExprType; int32_t GetBitmaskSize(int vocab_size) { return DynamicBitset::GetBufferSize(vocab_size); } DLDataType GetBitmaskDLType() { return DLDataType{kDLInt, 32, 1}; } namespace details { using Clock = std::chrono::steady_clock; using TimePoint = Clock::time_point; void ClearTokenBitmaskRow(int32_t* bitmask_data, int32_t bitmask_size, int32_t position) { std::fill_n(bitmask_data + position * bitmask_size, bitmask_size, 0); } bool TraverseDraftTreeRecursive( int32_t current_position, int32_t parent_position, const int64_t* retrieve_next_token, const int64_t* retrieve_next_sibling, const int64_t* draft_tokens, GrammarMatcher& matcher, DLTensor* token_bitmask, double time_threshold, const TimePoint& start_time ) { int32_t* bitmask_data = reinterpret_cast(token_bitmask->data); int32_t bitmask_size = static_cast(token_bitmask->shape[1]); bool accepted; if (current_position == 0) { // The first token generated by the target model is always accepted. accepted = true; } else { XGRAMMAR_CHECK(parent_position >= 0) << "Non-root draft tree nodes must have a valid parent position"; int64_t current_token_id = draft_tokens[current_position]; if (current_token_id < 0 || current_token_id >= static_cast(bitmask_size) * 32) { accepted = false; } else { int32_t* parent_bitmask = bitmask_data + parent_position * bitmask_size; // 32 boolean bitmask values are packed into 32-bit integers. uint32_t token_mask = uint32_t{1} << static_cast(current_token_id % 32); accepted = (static_cast(parent_bitmask[current_token_id / 32]) & token_mask) != 0; } // Check timeout for non-root nodes so the root token mask is still computed. if (accepted && time_threshold > 0) { auto elapsed = std::chrono::duration(Clock::now() - start_time).count(); if (elapsed > time_threshold) { return false; } } } if (accepted) { bool token_accepted = true; if (current_position != 0) { token_accepted = matcher.AcceptToken(static_cast(draft_tokens[current_position])); } if (token_accepted) { if (!matcher.IsTerminated()) { matcher.FillNextTokenBitmask(token_bitmask, current_position); if (retrieve_next_token[current_position] != -1) { bool success = TraverseDraftTreeRecursive( retrieve_next_token[current_position], current_position, retrieve_next_token, retrieve_next_sibling, draft_tokens, matcher, token_bitmask, time_threshold, start_time ); if (!success) { if (current_position != 0) { matcher.Rollback(1); } return false; } } } else { ClearTokenBitmaskRow(bitmask_data, bitmask_size, current_position); } if (current_position != 0) { matcher.Rollback(1); } } else { ClearTokenBitmaskRow(bitmask_data, bitmask_size, current_position); } } else { ClearTokenBitmaskRow(bitmask_data, bitmask_size, current_position); } if (retrieve_next_sibling[current_position] != -1) { bool success = TraverseDraftTreeRecursive( retrieve_next_sibling[current_position], parent_position, retrieve_next_token, retrieve_next_sibling, draft_tokens, matcher, token_bitmask, time_threshold, start_time ); if (!success) { return false; } } return true; } } // namespace details int32_t* CheckAndGetBitmaskPtr(const DLTensor& token_bitmask, int vocab_size, int index) { XGRAMMAR_CHECK(token_bitmask.dtype.code == kDLInt && token_bitmask.dtype.bits == 32) << "The provied bitmask's dtype is not valid: should be int32"; int32_t buffer_size = GetBitmaskSize(vocab_size); if (token_bitmask.ndim == 1) { XGRAMMAR_CHECK(token_bitmask.shape[0] == buffer_size) << "The provided bitmask's shape is not valid: should be (" << buffer_size << ", )"; XGRAMMAR_CHECK(index == 0) << "The index should be 0 when the bitmask is 1D"; } else { XGRAMMAR_CHECK(token_bitmask.ndim == 2) << "The provided bitmask's shape is not valid: should be (batch_size, " << buffer_size << ")"; XGRAMMAR_CHECK(token_bitmask.shape[1] == buffer_size) << "The provided bitmask's shape is not valid: should be (batch_size, " << buffer_size << ")"; XGRAMMAR_CHECK(index >= 0 && index < token_bitmask.shape[0]) << "The provided index is out of bounds"; } XGRAMMAR_CHECK( token_bitmask.device.device_type == kDLCPU || token_bitmask.device.device_type == kDLCUDAHost || token_bitmask.device.device_type == kDLROCMHost ) << "The provided bitmask's device is not valid: should be CPU"; return reinterpret_cast(token_bitmask.data) + index * buffer_size; } void _DebugGetMaskedTokensFromBitmask( std::vector* rejected_tokens, const DLTensor& token_bitmask, int vocab_size, int index ) { int32_t* data_ptr = CheckAndGetBitmaskPtr(token_bitmask, vocab_size, index); DynamicBitset bitset(vocab_size, reinterpret_cast(data_ptr)); rejected_tokens->clear(); for (int i = bitset.FindFirstZero(); i != -1; i = bitset.FindNextZero(i)) { rejected_tokens->push_back(i); } } std::pair _IsSingleTokenBitmask(const DLTensor& bitmask, int vocab_size, int index) { int32_t* data_ptr = CheckAndGetBitmaskPtr(bitmask, vocab_size, index); DynamicBitset bitset(vocab_size, reinterpret_cast(data_ptr)); if (bitset.Count() == 1) { return std::make_pair(true, bitset.FindFirstOne()); } else { return std::make_pair(false, -1); } } void ApplyMask32Bits( DLTensor* logits, const DLTensor& bitmask, int vocab_size, std::optional> indices ) { XGRAMMAR_CHECK(logits->dtype.code == kDLFloat && logits->dtype.bits == 32) << "The provided logits's dtype is not valid: should be float32"; std::pair logits_shape = logits->ndim == 2 ? std::make_pair(static_cast(logits->shape[0]), static_cast(logits->shape[1])) : std::make_pair(1, static_cast(logits->shape[0])); int logits_stride0 = logits->strides[0]; int bitmask_stride0 = bitmask.strides[0]; if (indices.has_value()) { for (auto idx : indices.value()) { uint32_t* data_ptr = reinterpret_cast(bitmask.data) + idx * bitmask_stride0; DynamicBitset bitset(vocab_size, data_ptr); auto logits_ptr = reinterpret_cast(logits->data) + idx * logits_stride0; for (int i = bitset.FindFirstZero(); i != -1; i = bitset.FindNextZero(i)) { logits_ptr[i] = -std::numeric_limits::infinity(); } } } else { for (int idx = 0; idx < logits_shape.first; ++idx) { uint32_t* data_ptr = reinterpret_cast(bitmask.data) + idx * bitmask_stride0; DynamicBitset bitset(vocab_size, data_ptr); auto logits_ptr = reinterpret_cast(logits->data) + idx * logits_stride0; for (int i = bitset.FindFirstZero(); i != -1; i = bitset.FindNextZero(i)) { logits_ptr[i] = -std::numeric_limits::infinity(); } } } } void ApplyMask16Bits( DLTensor* logits, const DLTensor& bitmask, int vocab_size, std::optional> indices ) { XGRAMMAR_CHECK(logits->dtype.bits == 16) << "The provided logits's dtype is not valid: should be bfloat16 or float16"; uint16_t kMinusInfinity; const uint16_t kMinusInfinityBf16 = 0xff80; const uint16_t kMinusInfinityFp16 = 0xfc00; switch (logits->dtype.code) { case kDLBfloat: kMinusInfinity = kMinusInfinityBf16; break; case kDLFloat: kMinusInfinity = kMinusInfinityFp16; break; default: XGRAMMAR_LOG(FATAL ) << "The provided logits's dtype is not valid: should be bfloat16 or float16"; } std::pair logits_shape = logits->ndim == 2 ? std::make_pair(static_cast(logits->shape[0]), static_cast(logits->shape[1])) : std::make_pair(1, static_cast(logits->shape[0])); int logits_stride0 = logits->strides[0]; int bitmask_stride0 = bitmask.strides[0]; if (indices.has_value()) { for (auto idx : indices.value()) { uint32_t* data_ptr = reinterpret_cast(bitmask.data) + idx * bitmask_stride0; DynamicBitset bitset(vocab_size, data_ptr); auto logits_ptr = reinterpret_cast(logits->data) + idx * logits_stride0; for (int i = bitset.FindFirstZero(); i != -1; i = bitset.FindNextZero(i)) { logits_ptr[i] = kMinusInfinity; } } } else { for (int idx = 0; idx < logits_shape.first; ++idx) { uint32_t* data_ptr = reinterpret_cast(bitmask.data) + idx * bitmask_stride0; DynamicBitset bitset(vocab_size, data_ptr); auto logits_ptr = reinterpret_cast(logits->data) + idx * logits_stride0; for (int i = bitset.FindFirstZero(); i != -1; i = bitset.FindNextZero(i)) { logits_ptr[i] = kMinusInfinity; } } } } void ApplyTokenBitmaskInplaceCPU( DLTensor* logits, const DLTensor& bitmask, int vocab_size, std::optional> indices ) { // Check device and dim XGRAMMAR_CHECK( logits->device.device_type == kDLCPU || logits->device.device_type == kDLCUDAHost || logits->device.device_type == kDLROCMHost ) << "The provided logits's device is not valid: should be CPU"; XGRAMMAR_CHECK( bitmask.device.device_type == kDLCPU || bitmask.device.device_type == kDLCUDAHost || bitmask.device.device_type == kDLROCMHost ) << "The provided bitmask's device is not valid: should be CPU"; XGRAMMAR_CHECK(logits->ndim == 2 || logits->ndim == 1) << "The provided logits's shape is not valid: should be 2D or 1D"; XGRAMMAR_CHECK(bitmask.ndim == 2 || bitmask.ndim == 1) << "The provided bitmask's shape is not valid: should be 2D or 1D"; // Check type XGRAMMAR_CHECK(logits->dtype.lanes == 1) << "The provided logits's dtype is not valid: lanes should be 1"; XGRAMMAR_CHECK( bitmask.dtype.code == kDLInt && bitmask.dtype.bits == 32 && bitmask.dtype.lanes == 1 ) << "The provided bitmask's dtype is not valid: should be int32"; // Check shape std::pair logits_shape = logits->ndim == 2 ? std::make_pair(static_cast(logits->shape[0]), static_cast(logits->shape[1])) : std::make_pair(1, static_cast(logits->shape[0])); std::pair bitmask_shape = bitmask.ndim == 2 ? std::make_pair(static_cast(bitmask.shape[0]), static_cast(bitmask.shape[1])) : std::make_pair(1, static_cast(bitmask.shape[0])); XGRAMMAR_CHECK( vocab_size <= bitmask_shape.second * DynamicBitset::BITS_PER_BLOCK && vocab_size <= logits_shape.second ); if (!indices.has_value()) { XGRAMMAR_CHECK(logits_shape.first == bitmask_shape.first) << "When indices is not provided, the logits's batch size should be equal to the " "bitmask's batch size, but got " << logits_shape.first << " vs " << bitmask_shape.first; } // Apply mask if (logits->dtype.bits == 32) { ApplyMask32Bits(logits, bitmask, vocab_size, indices); } else if (logits->dtype.bits == 16) { ApplyMask16Bits(logits, bitmask, vocab_size, indices); } else { XGRAMMAR_LOG(FATAL ) << "The provided logits's dtype is not valid: should be float32 or float16/bfloat16"; } } /******************* Grammar Matcher with Adaptive Token Mask *******************/ /* * Note on the matching algorithm (this is the old description for the matching algorithm, please * refer to https://arxiv.org/pdf/2411.15100 for the latest description) * * Given a context-free grammar, we match the characters in a string one by one. * * We adopt a non-deterministic pushdown automata (NPDA) in matching. To be specific, we maintain * several stacks, each of which represents a possible path in the NPDA, and update the stacks * during matching. * * ## Stack Structure (see grammar_matcher_state.h) * The element of every stack is a StackElement object, referring a position in the grammar. If a * StackElement points to a RuleRef element (referring to another rule), the next element of the * stack will be a position in this rule. If a StackElement is a CharacterClass element, it will be * the last in the stack, meaning *the next* character to match. * * ## Matching Process (see grammar_matcher_base.h) * When accepting a new character and it is accepted by a stack, the last element of the stack will * be advanced to the next position in the grammar. If it gets to the end of the rule, several * elements at the end may be popped out, and the last element of the stack will be advanced. * * One stack may split since there may be multiple possible next positions. In this case, similar * stacks with different top elements will be added. When one stack cannot accept the new character, * it will be removed from the stacks. * * ## Storage of Stacks (see grammar_matcher_state.h) * Note these stacks form a tree structure as when splitting, the new stacks share the same prefix. * We store all StackElements as a tree, where every path from tree root to a node represents a * stack. To represent stack tops, we attach additional pointers pointing the stack top nodes. * Also, We maintain a history of the stack top pointers, so we can rollback to the previous state. * * All tree nodes are maintained by a buffer, and utilize reference counting to recycle. If a node * is neither pointed by a stack top pointer, not pointed by some child nodes, it will be freed. * * ## Example * ### Grammar * root ::= [a] R * R ::= [b] S [c] | [b] [c] T * S ::= "" | [c] [d] * T ::= [e] * * ### The previous step * Previous accepted string: ab * Previous stack tree: * A------ * | \ \ * B D< E< * | * C< * * A: (rule root, choice 0, element 1) * B: (rule R, choice 0, element 1) * C: (rule S, choice 1, element 0) * D: (rule R, choice 0, element 2) * E: (rule R, choice 1, element 1) * < means the stack top pointers in the previous step. * The stacks in the previous step is: (A, B, C), (A, D), (A, E) * * ### The current step * Current accepted string: abc * Current stack tree: * A----------------- G<< * | \ \ \ * B--- D< E< H * | \ | * C< F<< I<< * * F: (rule S, choice 1, element 1) * G: (rule root, choice 0, element 2) (means the matching process has finished, and will be deleted * when the next char comes) * H: (rule R, choice 1, element 2) * I: (rule T, choice 0, element 0) * << means the stack top pointers in the current step. * The stacks in the current step is: (A, B, F), (A, H, I), (G,) * * ## Preprocess (see grammar_matcher_preproc.h) * We will store all information about tokens that needed in matching in a CompiledGrammar * object. Tokens are sorted by codepoint, allowing us to reuse the repeated prefixes between * different tokens. * * For a given position in a rule, if we only consider this rule and its sub-rules during matching, * without considering its parent rules (in actual matching, we also need to consider its parent * rules), we can already determine that some tokens are acceptable while others are definitely * rejected. Therefore, for a position in a rule, we can divide the token set into three categories: * - accepted_indices: If a token is accepted by this rule * - rejected_indices: If a token is rejected by this rule * - uncertain_indices: Whether it can be accepted depends on the information from the parent * level during actual matching. To be specific, If this token has a prefix that has not been * rejected and has reached the end of this rule, then it is possible for it to be further accepted * by the parent rule. * * During actual matching, we will directly accept or reject the tokens in accepted_indices and * rejected_indices, and only consider the tokens in uncertain_indices. That speeds up the matching * process. */ /* \brief The concrete implementation of GrammarMatcherNode. */ class GrammarMatcher::Impl : public EarleyParser { public: Impl( const CompiledGrammar& compiled_grammar, std::optional> override_stop_tokens = std::nullopt, bool terminate_without_stop_token = false, // max_rollback_tokens_ is deprecated and not used. int max_rollback_tokens = -1 ) : EarleyParser(compiled_grammar->grammar, ParserState::GetInvalidState()), compiled_grammar_(compiled_grammar), tokenizer_info_(compiled_grammar->tokenizer_info), stop_token_ids_(override_stop_tokens.value_or(tokenizer_info_.GetStopTokenIds())), terminate_without_stop_token_(terminate_without_stop_token), tmp_accepted_bitset_(tokenizer_info_.GetVocabSize()) { XGRAMMAR_CHECK(!override_stop_tokens.has_value() || !override_stop_tokens->empty()) << "The override_stop_tokens should not be empty"; } bool AcceptToken(int32_t token_id, bool debug_print = false); bool AcceptString(const std::string& input_str, bool debug_print = false); bool FillNextTokenBitmask(DLTensor* next_token_bitmask, int index, bool debug_print = false); std::string FindJumpForwardString(); void Rollback(int num_tokens); bool IsTerminated() const; void Reset() { EarleyParser::Reset(); } int GetMaxRollbackTokens() const { return -1; } const std::vector& GetStopTokenIds() const { return stop_token_ids_; } std::string _DebugPrintInternalState() const { return PrintStates(); } private: using StoreType = AdaptiveTokenMask::StoreType; /*! * \brief If is_uncertain_saved is true, find the next token in uncertain_indices. Otherwise, * find the next token that is set to true in uncertain_tokens_bitset. * \param iterator_uncertain The helper iterator to iterate over uncertain_indices or * uncertain_tokens_bitset. * \returns The index of the next token, or -1 if no more token. */ int GetNextUncertainToken( bool is_uncertain_saved, int* iterator_uncertain, const std::vector& uncertain_indices, const std::vector& uncertain_tokens_bitset ); /*! \brief Set the acceptable next token in next_token_bitmask. */ void SetTokenBitmask( int32_t* bitmask_data_ptr, const DynamicBitset& accepted_bitset, const std::vector& rejected_indices, bool can_reach_end, bool allow_special_token = false ); /*! * \brief Accept the stop token and terminates the matcher. * \returns Whether the stop token can be accepted. */ bool AcceptStopToken(); bool IsStopTokenAccepted() const; /*! \brief Check if the token bitmask is all-true. */ bool IsTokenBitmaskAllTrue(int32_t* bitmask_data_ptr); std::string PrintBitmask(int32_t* bitmask_data_ptr, const TokenizerInfo& tokenizer_info); CompiledGrammar compiled_grammar_; TokenizerInfo tokenizer_info_; std::vector stop_token_ids_; bool terminate_without_stop_token_; std::deque token_length_history; // Temporary data for FillNextTokenBitmask. They are stored here to avoid repeated allocation. DynamicBitset tmp_accepted_bitset_; std::vector tmp_rejected_indices_; std::vector tmp_rejected_indices_delta_; }; class BatchGrammarMatcher::Impl { public: Impl(std::variant max_threads) { if (std::holds_alternative(max_threads)) { int32_t num_threads = std::get(max_threads); XGRAMMAR_CHECK(num_threads >= 1) << "The num_threads should be at least 1, but got " << num_threads; if (num_threads > 1) { if (num_threads > static_cast(std::thread::hardware_concurrency())) { XGRAMMAR_LOG(WARNING) << "The num_threads " << num_threads << " is larger than the " << "number of hardware threads. Using " << static_cast(std::thread::hardware_concurrency()) << " instead."; } max_threads_ = std::min(num_threads, static_cast(std::thread::hardware_concurrency())); } } else { std::string str = std::get(max_threads); XGRAMMAR_CHECK(str == "auto"); max_threads_ = std::thread::hardware_concurrency() / 2; } } void BatchFillNextTokenBitmask( std::vector* matchers, DLTensor* next_token_bitmask, const std::optional>& indices, bool debug_print ); static std::vector BatchAcceptToken( std::vector* matchers, const std::vector& token_ids, bool debug_print ); static std::vector BatchAcceptString( std::vector* matchers, const std::vector& input_strs, bool debug_print ); static void BatchRollback( std::vector* matchers, const std::vector& num_tokens ); private: std::optional thread_pool_ = std::nullopt; int32_t max_threads_ = 1; }; bool GrammarMatcher::Impl::AcceptStopToken() { if (terminate_without_stop_token_) { return false; } if (!IsCompleted()) { return false; } XGRAMMAR_DCHECK(!stop_token_is_accepted_); token_length_history.push_back(0); stop_token_is_accepted_ = true; return true; } bool GrammarMatcher::Impl::IsTerminated() const { if (terminate_without_stop_token_) { return IsCompleted(); } return IsStopTokenAccepted(); } bool GrammarMatcher::Impl::IsStopTokenAccepted() const { return stop_token_is_accepted_; } // TODO(yixin): Polish verbose logging bool GrammarMatcher::Impl::AcceptToken(int32_t token_id, bool debug_print) { if (IsStopTokenAccepted()) { XGRAMMAR_LOG(WARNING) << "The matcher has terminated after accepting the stop token, but is " << "trying to accept new token with id " << token_id << "."; return false; } if (token_id < 0 || token_id >= tokenizer_info_.GetVocabSize()) { XGRAMMAR_LOG(WARNING) << "The token id " << token_id << " is out of range [0, " << tokenizer_info_.GetVocabSize() << "). Rejecting the token."; return false; } if (debug_print) { std::string states_str; for (const auto& state : GetLatestScanableStates()) { states_str += " " + state.ToString() + "\n"; } XGRAMMAR_LOG(INFO) << "Accepting token id " << token_id << ", string: \"" << EscapeString(tokenizer_info_.GetDecodedVocab()[token_id]) << "\", current state:\n" << states_str; } // Handle the stop token if (std::find(stop_token_ids_.begin(), stop_token_ids_.end(), token_id) != stop_token_ids_.end()) { bool accepted = AcceptStopToken(); if (debug_print) { XGRAMMAR_LOG(INFO) << "The token is an end token. Is accepted: " << accepted; } return accepted; } const auto& special_token_ids = tokenizer_info_.GetSpecialTokenIds(); if (std::find(special_token_ids.begin(), special_token_ids.end(), token_id) != special_token_ids.end()) { XGRAMMAR_LOG(WARNING) << "GrammarMatcher cannot accept special token id " << token_id << ": " << tokenizer_info_.GetDecodedVocab()[token_id] << ". Rejecting the token."; return false; } const auto& token = tokenizer_info_.GetDecodedVocab()[token_id]; // Phase 1: Try atomic token path (from current state, before byte path) std::vector atomic_states; std::vector> atomic_completable; bool atomic_completed = false; bool atomic_success = AdvanceAtomicToken(token_id, debug_print); if (atomic_success) { atomic_states = GetLatestScanableStates(); auto row = rule_id_to_completable_states_.Back(); atomic_completable.assign(row.data, row.data + row.data_len); atomic_completed = is_completed_.back(); PopLastStates(1); } // Phase 2: Try byte-by-byte path (from the same original state) int pos = 0; bool byte_path_success = true; for (auto char_value : token) { if (!Advance(char_value, debug_print)) { byte_path_success = false; break; } ++pos; } // Phase 3: Combine results (no priority — merge with deduplication) if (!byte_path_success && !atomic_success) { if (debug_print) { XGRAMMAR_LOG(INFO) << "Token #" << token_id << "<" << EscapeString(token) << "> rejected at position " << pos; } PopLastStates(pos); return false; } if (atomic_success && !byte_path_success) { PopLastStates(pos); AdvanceAtomicToken(token_id, debug_print); token_length_history.push_back(1); } else if (byte_path_success && !atomic_success) { token_length_history.push_back(token.size()); } else { // Both paths succeeded — merge atomic token states into byte path if (token.empty()) { // Zero-length token: byte path created 0 timepoints, just push atomic states scanable_state_history_.PushBack(atomic_states); rule_id_to_completable_states_.PushBack(atomic_completable); is_completed_.push_back(atomic_completed); token_length_history.push_back(1); } else { auto byte_states = GetLatestScanableStates(); std::vector merged = byte_states; StateEqualForParsing state_eq; for (const auto& s : atomic_states) { if (std::find_if(merged.begin(), merged.end(), [&](const auto& m) { return state_eq(m, s); }) == merged.end()) { merged.push_back(s); } } auto byte_row = rule_id_to_completable_states_.Back(); std::vector> merged_completable( byte_row.data, byte_row.data + byte_row.data_len ); bool byte_completed = is_completed_.back(); PopLastStates(1); for (const auto& cs : atomic_completable) { if (std::find_if(merged_completable.begin(), merged_completable.end(), [&](const auto& m) { return m.first == cs.first && state_eq(m.second, cs.second); }) == merged_completable.end()) { merged_completable.push_back(cs); } } scanable_state_history_.PushBack(merged); rule_id_to_completable_states_.PushBack(merged_completable); is_completed_.push_back(byte_completed || atomic_completed); token_length_history.push_back(token.size()); } } if (debug_print) { XGRAMMAR_LOG(INFO) << "Token #" << token_id << "<" << EscapeString(token) << "> accepted."; } return true; } bool GrammarMatcher::Impl::AcceptString(const std::string& input_str, bool debug_print) { if (IsStopTokenAccepted()) { XGRAMMAR_LOG(WARNING) << "The matcher has terminated after accepting the stop token, but is " << "trying to accept new string \"" << EscapeString(input_str) << "\"."; return false; } if (debug_print) { XGRAMMAR_LOG(INFO) << "Trying to accept string \"" << EscapeString(input_str) << "\". Current state:\n" << PrintStates(); } int accepted_cnt = 0; for (auto char_value : input_str) { if (!Advance(char_value, debug_print)) { if (debug_print) { XGRAMMAR_LOG(INFO) << "String \"" << EscapeString(input_str) << "\" is rejected at " << "position " << accepted_cnt << ", char " << EscapeString(char_value); } PopLastStates(accepted_cnt); return false; } if (debug_print) { XGRAMMAR_LOG(INFO) << "Char " << EscapeString(char_value) << " is accepted. Current state:\n" << PrintStates(); } ++accepted_cnt; } token_length_history.push_back(input_str.size()); if (debug_print) { XGRAMMAR_LOG(INFO) << "String \"" << EscapeString(input_str) << "\" is accepted."; } return true; } std::string GrammarMatcher::Impl::PrintBitmask( int32_t* bitmask_data_ptr, const TokenizerInfo& tokenizer_info ) { constexpr int kMaxPrintTokens = 100; std::vector accepted_ids; std::vector rejected_ids; auto bitset = DynamicBitset(tokenizer_info.GetVocabSize(), reinterpret_cast(bitmask_data_ptr)); for (int i = 0; i < tokenizer_info.GetVocabSize(); ++i) { if (bitset[i]) { accepted_ids.push_back(i); } else { rejected_ids.push_back(i); } } std::stringstream ss; ss << "TokenBitmask(num_tokens=" << tokenizer_info.GetVocabSize() << ", accepted_num=" << accepted_ids.size() << ", rejected_num=" << rejected_ids.size() << ",\naccepted_ids=" << PrintTokenByIds(accepted_ids, tokenizer_info, kMaxPrintTokens) << ",\nrejected_ids=" << PrintTokenByIds(rejected_ids, tokenizer_info, kMaxPrintTokens) << ")"; return ss.str(); } bool GrammarMatcher::Impl::IsTokenBitmaskAllTrue(int32_t* bitmask_data_ptr) { DynamicBitset next_token_bitset( tokenizer_info_.GetVocabSize(), reinterpret_cast(bitmask_data_ptr) ); return next_token_bitset.All(); } bool GrammarMatcher::Impl::FillNextTokenBitmask( DLTensor* next_token_bitmask, int index, bool debug_print ) { XGRAMMAR_CHECK(!IsStopTokenAccepted()) << "GrammarMatcher has terminated after accepting the stop token, but is trying to " "find the next token mask"; int32_t* bitmask_data_ptr = CheckAndGetBitmaskPtr(*next_token_bitmask, tokenizer_info_.GetVocabSize(), index); const auto& sorted_decoded_vocab = tokenizer_info_.GetSortedDecodedVocab(); const auto& subtree_range = tokenizer_info_.GetTrieSubtreeNodesRange(); const auto& adaptive_token_mask_cache = compiled_grammar_->adaptive_token_mask_cache; // We need to have a copy, because scanable_state_history_ will be modified during the // FillNextTokenBitmask process, which can lead to undefined behavior. auto latest_states = GetLatestScanableStates(); // We check all the latest states of the earley parser, and check all the masks of the leaf // states. The final accepted token set is the union of the accepted token sets of all leaf // states. The final rejected token set is the intersection of the rejected token sets of all leaf // states. // Note these indices store the indices in sorted_decoded_vocab, instead of the token ids. tmp_accepted_bitset_.Reset(); // {-1} means the universal set, i.e. all tokens initially tmp_rejected_indices_.assign({-1}); if (debug_print) { XGRAMMAR_LOG(INFO) << "FillNextTokenBitmask: index=" << index << ", num of states=" << latest_states.size(); } std::vector> latest_states_with_masks; for (const auto& state : latest_states) { auto adaptive_token_mask_it = adaptive_token_mask_cache.find(state); XGRAMMAR_CHECK(adaptive_token_mask_it != adaptive_token_mask_cache.end()) << state; const auto& adaptive_token_mask = adaptive_token_mask_it->second; latest_states_with_masks.push_back(std::make_pair(state, adaptive_token_mask_it)); if (adaptive_token_mask.store_type == StoreType::kAcceptedBitset) { tmp_accepted_bitset_ |= adaptive_token_mask.accepted_bitset; } else if (adaptive_token_mask.store_type == StoreType::kAccepted) { for (auto idx : adaptive_token_mask.accepted_indices) { tmp_accepted_bitset_.Set(sorted_decoded_vocab[idx].first, true); } } } for (const auto& [state, adaptive_token_mask_it] : latest_states_with_masks) { const auto& adaptive_token_mask = adaptive_token_mask_it->second; // For each ParserState, we will check every uncertain token and put them into the accepted or // rejected list. // Step 2. Update the accepted tokens in accepted_indices_delta, or the rejected tokens in // rejected_indices_delta. // If the accepted tokens are saved, it means it is likely to be smaller than the rejected // tokens, so we will just find the accepted tokens, and vice versa. tmp_rejected_indices_delta_.clear(); // Examine only the current one ParserState PushOneStateToCheck(state); const std::string* prev_token = nullptr; int prev_matched_size = 0; if (debug_print) { XGRAMMAR_LOG(INFO) << "The ParserState is " << state << ", the mask is " << adaptive_token_mask.Print(tokenizer_info_); } int last_rejected_uncertain_range = 0; for (const auto& cur_token_idx : adaptive_token_mask.uncertain_indices) { // Check if the current token is already accepted. If it is, we can skip it. if (tmp_accepted_bitset_[sorted_decoded_vocab[cur_token_idx].first]) { continue; } // Check if the current token is in the rejected range. i.e. check if the current token // is on the subtree of the rejected token. if (cur_token_idx < last_rejected_uncertain_range) { if (adaptive_token_mask.store_type == StoreType::kRejected) { tmp_rejected_indices_delta_.push_back(cur_token_idx); } continue; } const auto& cur_token = sorted_decoded_vocab[cur_token_idx].second; bool accepted = true; // Step 2.1. Find the longest common prefix with the accepted part of the previous token. // We can reuse the previous matched size to avoid unnecessary matching. if (prev_token) { int lcp_len = std::mismatch( cur_token.begin(), cur_token.end(), prev_token->begin(), prev_token->end() ) .first - cur_token.begin(); if (lcp_len > prev_matched_size) { last_rejected_uncertain_range = subtree_range[cur_token_idx]; accepted = false; } else if (lcp_len < prev_matched_size) { PopLastStates(prev_matched_size - lcp_len); } prev_matched_size = std::min(prev_matched_size, lcp_len); } // Step 2.2. Find if the current token is accepted or rejected. if (accepted) { for (int j = prev_matched_size; j < static_cast(cur_token.size()); ++j) { if (!Advance(cur_token[j])) { last_rejected_uncertain_range = subtree_range[cur_token_idx]; accepted = false; break; } prev_matched_size = j + 1; } } // Step 2.3. Push the result to the delta list. if (adaptive_token_mask.store_type == StoreType::kAcceptedBitset || adaptive_token_mask.store_type == StoreType::kAccepted) { if (accepted) { tmp_accepted_bitset_.Set(sorted_decoded_vocab[cur_token_idx].first, true); } } else { if (!accepted) { tmp_rejected_indices_delta_.push_back(cur_token_idx); } } prev_token = &cur_token; } PopLastStates(prev_matched_size + 1); // Step 3. Update the accepted_indices or rejected_indices if (adaptive_token_mask.store_type == StoreType::kRejected) { // rejected_indices = Intersect( // rejected_indices, // adaptive_token_mask.rejected_indices + rejected_indices_delta) IntsetUnion(&tmp_rejected_indices_delta_, adaptive_token_mask.rejected_indices); IntsetIntersection(&tmp_rejected_indices_, tmp_rejected_indices_delta_); } } // Finally update the rejected_ids bitset bool can_reach_end = IsCompleted(); SetTokenBitmask( bitmask_data_ptr, tmp_accepted_bitset_, tmp_rejected_indices_, can_reach_end, false ); if (debug_print) { XGRAMMAR_LOG(INFO) << "Filled bitmask: " << PrintBitmask(bitmask_data_ptr, tokenizer_info_); } return !IsTokenBitmaskAllTrue(bitmask_data_ptr); } std::string GrammarMatcher::Impl::FindJumpForwardString() { XGRAMMAR_CHECK(!IsStopTokenAccepted()) << "GrammarMatcher has terminated after accepting the stop token, but is trying to " "get the jump forward string"; std::string result; int num_accepted_chars = 0; bool can_find_next_char = true; while (can_find_next_char) { const auto& states = scanable_state_history_[scanable_state_history_.size() - 1]; // The state comes to the end of the grammar if (IsCompleted()) { can_find_next_char = false; break; } // 1. Check that for every leaf ParserState, the next possible char is unique and the same // -1 means not found yet; 0~255 means the next char int next_char = -1; for (const auto& state : states) { XGRAMMAR_DCHECK(state.rule_id != -1 && grammar_->per_rule_fsms[state.rule_id].has_value()); const auto& fsm = grammar_->per_rule_fsms[state.rule_id].value(); const auto& current_edges = fsm.GetFsm().GetFsm().GetEdges(state.element_id); for (const auto& edge : current_edges) { if (!edge.IsCharRange()) { continue; } if (edge.min != edge.max) { can_find_next_char = false; break; } if (next_char == -1) { next_char = edge.min; } else if (next_char != edge.min) { can_find_next_char = false; break; } } } if (next_char == -1) { can_find_next_char = false; } // 2. If found, accept the char and iterate to the next position if (can_find_next_char) { result += static_cast(next_char); Advance(next_char); ++num_accepted_chars; } } // Rollback all chars accepted PopLastStates(num_accepted_chars); return result; } void GrammarMatcher::Impl::Rollback(int num_tokens) { XGRAMMAR_CHECK(num_tokens <= static_cast(token_length_history.size())) << "Intended to rollback " << num_tokens << " tokens, but only the last " << token_length_history.size() << " steps of history are saved"; while (num_tokens > 0) { int steps = token_length_history.back(); PopLastStates(steps); token_length_history.pop_back(); --num_tokens; } } void GrammarMatcher::Impl::SetTokenBitmask( int32_t* bitmask_data_ptr, const DynamicBitset& accepted_bitset, const std::vector& rejected_indices, bool can_reach_end, bool allow_special_token ) { // next_token_bitmask = set(all accepted tokens) = // 1. all_tokens - (rejected_ids / accepted_ids) // (when rejected_ids != {-1}, i.e. rejected_ids is not the universal set) // 2. accepted_ids // (otherwise, when rejected_ids is the universal set) DynamicBitset next_token_bitset( tokenizer_info_.GetVocabSize(), reinterpret_cast(bitmask_data_ptr) ); const auto& sorted_decoded_vocab = tokenizer_info_.GetSortedDecodedVocab(); if (rejected_indices.size() == 1 && rejected_indices[0] == -1) { // If rejected_indices is the universal set, the final accepted token set is just // accepted_indices next_token_bitset = accepted_bitset; if (allow_special_token) { for (int id : tokenizer_info_.GetSpecialTokenIds()) { next_token_bitset.Set(id, true); } } if (can_reach_end) { // add end tokens for (int id : stop_token_ids_) { next_token_bitset.Set(id, true); } } } else { // Otherwise, the final rejected token set is (rejected_indices \ accepted_indices) next_token_bitset.Set(); for (auto i : rejected_indices) { auto id = sorted_decoded_vocab[i].first; if (!accepted_bitset[id]) { next_token_bitset.Set(id, false); } } if (!allow_special_token) { for (int id : tokenizer_info_.GetSpecialTokenIds()) { next_token_bitset.Set(id, false); } } if (!can_reach_end) { for (int id : stop_token_ids_) { next_token_bitset.Set(id, false); } } } } int GrammarMatcher::Impl::GetNextUncertainToken( bool is_uncertain_saved, int* iterator_uncertain, const std::vector& uncertain_indices, const std::vector& uncertain_tokens_bitset ) { if (is_uncertain_saved) { ++*iterator_uncertain; if (*iterator_uncertain == static_cast(uncertain_indices.size())) { return -1; } return uncertain_indices[*iterator_uncertain]; } else { ++*iterator_uncertain; while (*iterator_uncertain < static_cast(uncertain_tokens_bitset.size()) && !uncertain_tokens_bitset[*iterator_uncertain]) { ++*iterator_uncertain; } if (*iterator_uncertain == static_cast(uncertain_tokens_bitset.size())) { return -1; } return *iterator_uncertain; } } void BatchGrammarMatcher::Impl::BatchFillNextTokenBitmask( std::vector* matchers, DLTensor* next_token_bitmask, const std::optional>& indices, bool debug_print ) { XGRAMMAR_CHECK(!indices.has_value() || indices->size() == matchers->size()) << "The size of indices (" << (indices.has_value() ? indices->size() : 0) << ") should be the same as the size of matchers (" << matchers->size() << ")."; // Initialize the thread pool if needed. It should be initialized each time, // because ThreadPool cannot be reused after Join(). if (max_threads_ > 1) { thread_pool_.emplace(max_threads_); } if (!thread_pool_.has_value()) { for (int i = 0; i < static_cast(matchers->size()); i++) { auto& matcher = (*matchers)[i]; int index = indices.has_value() ? (*indices)[i] : i; XGRAMMAR_CHECK(index >= 0 && index < next_token_bitmask->shape[0]) << "The index " << index << " is out of range [0, " << next_token_bitmask->shape[0] << ") for batch_id " << i << "."; matcher->FillNextTokenBitmask(next_token_bitmask, index, debug_print); } } else { auto fill_next_token_mask = [&](int32_t batch_id) { auto& matcher = (*matchers)[batch_id]; int index = indices.has_value() ? (*indices)[batch_id] : batch_id; XGRAMMAR_CHECK(index >= 0 && index < next_token_bitmask->shape[0]) << "The index " << index << " is out of range [0, " << next_token_bitmask->shape[0] << ") for batch_id " << batch_id << "."; matcher->FillNextTokenBitmask(next_token_bitmask, index, debug_print); }; for (int i = 0; i < static_cast(matchers->size()); i++) { thread_pool_->Execute([fill_next_token_mask, i]() { fill_next_token_mask(i); }); } thread_pool_->Join(); } } std::vector BatchGrammarMatcher::Impl::BatchAcceptString( std::vector* matchers, const std::vector& input_strs, bool debug_print ) { XGRAMMAR_CHECK(matchers->size() == input_strs.size()) << "The size of matchers (" << matchers->size() << ") and input_strs (" << input_strs.size() << ") should be the same."; std::vector accepted(matchers->size()); for (int i = 0; i < static_cast(matchers->size()); i++) { auto& matcher = (*matchers)[i]; accepted[i] = matcher->AcceptString(input_strs[i], debug_print); } return accepted; } std::vector BatchGrammarMatcher::Impl::BatchAcceptToken( std::vector* matchers, const std::vector& token_ids, bool debug_print ) { XGRAMMAR_CHECK(matchers->size() == token_ids.size()) << "The size of matchers (" << matchers->size() << ") and token_ids (" << token_ids.size() << ") should be the same."; std::vector accepted(matchers->size()); for (int i = 0; i < static_cast(matchers->size()); i++) { auto& matcher = (*matchers)[i]; accepted[i] = matcher->AcceptToken(token_ids[i], debug_print); } return accepted; } void BatchGrammarMatcher::Impl::BatchRollback( std::vector* matchers, const std::vector& num_tokens ) { XGRAMMAR_CHECK(matchers->size() == num_tokens.size()) << "The size of matchers (" << matchers->size() << ") and num_tokens (" << num_tokens.size() << ") should be the same."; for (int i = 0; i < static_cast(matchers->size()); i++) { (*matchers)[i].Rollback(num_tokens[i]); } } GrammarMatcher::GrammarMatcher( const CompiledGrammar& compiled_grammar, std::optional> override_stop_tokens, bool terminate_without_stop_token, int max_rollback_tokens ) : pimpl_(std::make_shared( compiled_grammar, override_stop_tokens, terminate_without_stop_token, max_rollback_tokens )) {} bool GrammarMatcher::AcceptToken(int32_t token_id, bool debug_print) { return pimpl_->AcceptToken(token_id, debug_print); } bool GrammarMatcher::AcceptString(const std::string& input_str, bool debug_print) { return pimpl_->AcceptString(input_str, debug_print); } bool GrammarMatcher::FillNextTokenBitmask( DLTensor* next_token_bitmask, int index, bool debug_print ) { return pimpl_->FillNextTokenBitmask(next_token_bitmask, index, debug_print); } bool GrammarMatcher::TraverseDraftTree( const DLTensor* retrieve_next_token, const DLTensor* retrieve_next_sibling, const DLTensor* draft_tokens, DLTensor* token_bitmask, double time_threshold ) { auto check_cpu = [](const DLTensor* tensor, const char* name) { XGRAMMAR_CHECK( tensor->device.device_type == kDLCPU || tensor->device.device_type == kDLCUDAHost || tensor->device.device_type == kDLROCMHost ) << "The " << name << " tensor must be on CPU"; }; XGRAMMAR_CHECK( retrieve_next_token->ndim == 1 && retrieve_next_token->dtype.code == kDLInt && retrieve_next_token->dtype.bits == 64 ) << "The retrieve_next_token tensor must be a 1D int64 tensor"; XGRAMMAR_CHECK( retrieve_next_sibling->ndim == 1 && retrieve_next_sibling->dtype.code == kDLInt && retrieve_next_sibling->dtype.bits == 64 ) << "The retrieve_next_sibling tensor must be a 1D int64 tensor"; XGRAMMAR_CHECK( draft_tokens->ndim == 1 && draft_tokens->dtype.code == kDLInt && draft_tokens->dtype.bits == 64 ) << "The draft_tokens tensor must be a 1D int64 tensor"; XGRAMMAR_CHECK( token_bitmask->ndim == 2 && token_bitmask->dtype.code == kDLInt && token_bitmask->dtype.bits == 32 ) << "The token_bitmask tensor must be a 2D int32 tensor"; check_cpu(retrieve_next_token, "retrieve_next_token"); check_cpu(retrieve_next_sibling, "retrieve_next_sibling"); check_cpu(draft_tokens, "draft_tokens"); check_cpu(token_bitmask, "token_bitmask"); XGRAMMAR_CHECK(retrieve_next_token->shape[0] == retrieve_next_sibling->shape[0]) << "The retrieve_next_token and retrieve_next_sibling tensors must have the same length"; XGRAMMAR_CHECK(retrieve_next_token->shape[0] == draft_tokens->shape[0]) << "The retrieve_next_token and draft_tokens tensors must have the same length"; XGRAMMAR_CHECK(retrieve_next_token->shape[0] == token_bitmask->shape[0]) << "The token_bitmask batch size must match the number of nodes in the tree"; XGRAMMAR_CHECK(retrieve_next_sibling->shape[0] > 0 && retrieve_next_sibling->data != nullptr) << "The draft tree must not be empty"; XGRAMMAR_CHECK(reinterpret_cast(retrieve_next_sibling->data)[0] == -1) << "The root node must not have siblings"; return details::TraverseDraftTreeRecursive( 0, -1, reinterpret_cast(retrieve_next_token->data), reinterpret_cast(retrieve_next_sibling->data), reinterpret_cast(draft_tokens->data), *this, token_bitmask, time_threshold, details::Clock::now() ); } std::string GrammarMatcher::FindJumpForwardString() { return pimpl_->FindJumpForwardString(); } void GrammarMatcher::Rollback(int num_tokens) { pimpl_->Rollback(num_tokens); } bool GrammarMatcher::IsTerminated() const { return pimpl_->IsTerminated(); } bool GrammarMatcher::IsCompleted() const { return pimpl_->IsCompleted(); } void GrammarMatcher::Reset() { pimpl_->Reset(); } GrammarMatcher GrammarMatcher::Fork() const { return GrammarMatcher(std::make_shared(*pimpl_)); } int GrammarMatcher::GetMaxRollbackTokens() const { return pimpl_->GetMaxRollbackTokens(); } const std::vector& GrammarMatcher::GetStopTokenIds() const { return pimpl_->GetStopTokenIds(); } std::string GrammarMatcher::_DebugPrintInternalState() const { return pimpl_->_DebugPrintInternalState(); } void BatchGrammarMatcher::BatchFillNextTokenBitmask( std::vector* matchers, DLTensor* next_token_bitmask, const std::optional>& indices, bool debug_print ) { return pimpl_->BatchFillNextTokenBitmask(matchers, next_token_bitmask, indices, debug_print); } std::vector BatchGrammarMatcher::BatchAcceptString( std::vector* matchers, const std::vector& input_strs, bool debug_print ) { return Impl::BatchAcceptString(matchers, input_strs, debug_print); } std::vector BatchGrammarMatcher::BatchAcceptToken( std::vector* matchers, const std::vector& token_ids, bool debug_print ) { return Impl::BatchAcceptToken(matchers, token_ids, debug_print); } void BatchGrammarMatcher::BatchRollback( std::vector* matchers, const std::vector& num_tokens ) { Impl::BatchRollback(matchers, num_tokens); } BatchGrammarMatcher::BatchGrammarMatcher(std::variant max_threads) : pimpl_(std::make_shared(max_threads)) {} } // namespace xgrammar xgrammar-0.2.3/cpp/grammar_parser.cc000066400000000000000000001146541521764210300174410ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar_parser.cc */ #include "grammar_parser.h" #include #include #include #include #include #include #include "grammar_builder.h" #include "grammar_impl.h" #include "support/encoding.h" #include "support/logging.h" #include "xgrammar/grammar.h" namespace xgrammar { class EBNFLexer::Impl { public: using Token = EBNFLexer::Token; using TokenType = EBNFLexer::TokenType; std::vector Tokenize(const std::string& input); private: std::string input_; const char* cur_ = nullptr; int cur_line_ = 1; int cur_column_ = 1; constexpr static int64_t kMaxIntegerInGrammar = 1e15; // Helper functions /*! * \brief Consume a character sequence and return the next token. Return a token if it's a * single token, or a vector of tokens if it's a sequence of tokens. * * \return std::variant> */ std::variant> NextToken(); Token ParseIdentifierOrBooleanToken(); Token ParseStringToken(); std::vector ParseCharClassToken(); Token ParseIntegerToken(); [[noreturn]] void ReportLexerError(const std::string& msg, int line = -1, int column = -1); char Peek(int delta = 0) const; void Consume(int cnt = 1); void ConsumeSpace(); std::string ParseIdentifierToken(); void ConvertIdentifierToRuleName(std::vector* tokens); static bool IsNameChar(char c, bool is_first = false); }; // Look at the next character inline char EBNFLexer::Impl::Peek(int delta) const { return *(cur_ + delta); } // Consume characters and update position information inline void EBNFLexer::Impl::Consume(int cnt) { for (int i = 0; i < cnt; ++i) { // Newline\n \r \r\n if (*cur_ == '\n' || (*cur_ == '\r' && *(cur_ + 1) != '\n')) { ++cur_line_; cur_column_ = 1; } else { ++cur_column_; } ++cur_; } } // Skip whitespace and comments void EBNFLexer::Impl::ConsumeSpace() { while (Peek() && (Peek() == ' ' || Peek() == '\t' || Peek() == '#' || Peek() == '\n' || Peek() == '\r')) { Consume(); if (Peek(-1) == '#') { while (Peek() && Peek() != '\n' && Peek() != '\r') { Consume(); } if (!Peek()) { return; } Consume(); if (Peek(-1) == '\r' && Peek() == '\n') { Consume(); } } } } // Report parsing error void EBNFLexer::Impl::ReportLexerError(const std::string& msg, int line, int column) { int line_to_print = line == -1 ? cur_line_ : line; int column_to_print = column == -1 ? cur_column_ : column; XGRAMMAR_LOG(FATAL) << "EBNF lexer error at line " + std::to_string(line_to_print) + ", column " + std::to_string(column_to_print) + ": " + msg; XGRAMMAR_UNREACHABLE(); } // Check if a character can be part of an identifier bool EBNFLexer::Impl::IsNameChar(char c, bool is_first) { return c == '_' || c == '-' || c == '.' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (!is_first && c >= '0' && c <= '9'); } // Parse identifier std::string EBNFLexer::Impl::ParseIdentifierToken() { const char* start = cur_; bool first_char = true; while (*cur_ && IsNameChar(*cur_, first_char)) { Consume(); first_char = false; } if (start == cur_) { ReportLexerError("Expect identifier"); } return std::string(start, cur_ - start); } // Parse identifier or boolean value EBNFLexer::Token EBNFLexer::Impl::ParseIdentifierOrBooleanToken() { int start_line = cur_line_; int start_column = cur_column_; std::string identifier = ParseIdentifierToken(); // Check if it's a boolean value if (identifier == "true" || identifier == "false") { return { TokenType::BooleanLiteral, identifier, identifier == "true" ? true : false, start_line, start_column }; } // Otherwise it's an identifier return {TokenType::Identifier, identifier, identifier, start_line, start_column}; } // Parse string literal EBNFLexer::Token EBNFLexer::Impl::ParseStringToken() { int start_line = cur_line_; int start_column = cur_column_; const char* start_pos = cur_; Consume(); // Skip opening quote std::vector codepoints; while (Peek() && Peek() != '"' && Peek() != '\n' && Peek() != '\r') { auto [codepoint, len] = ParseNextUTF8OrEscaped(cur_); if (codepoint == CharHandlingError::kInvalidUTF8) { ReportLexerError("Invalid UTF8 sequence"); } if (codepoint == CharHandlingError::kInvalidEscape) { ReportLexerError("Invalid escape sequence"); } Consume(len); codepoints.push_back(codepoint); } if (Peek() != '"') { ReportLexerError("Expect \" in string literal"); } Consume(); // Skip closing quote // Extract original lexeme std::string lexeme(start_pos, cur_ - start_pos); // Convert codepoints to UTF-8 string value std::string value; for (auto codepoint : codepoints) { value += CharToUTF8(codepoint); } return {TokenType::StringLiteral, lexeme, value, start_line, start_column}; } // Parse character class. std::vector EBNFLexer::Impl::ParseCharClassToken() { std::vector tokens; tokens.push_back({TokenType::LBracket, "[", "", cur_line_, cur_column_}); Consume(); // Skip '[' if (Peek() == '^') { tokens.push_back({TokenType::Caret, "^", "", cur_line_, cur_column_}); Consume(); } static const std::unordered_map kRegexEscapeChars = { // clang-format off {'^', '^'}, {'$', '$'}, {'\\', '\\'}, {'.', '.'}, {'*', '*'}, {'+', '+'}, {'?', '?'}, {'(', '('}, {')', ')'}, {'[', '['}, {']', ']'}, {'{', '{'}, {'}', '}'}, {'|', '|'}, {'/', '/'}, {'-', '-'} // clang-format on }; static const std::unordered_set kRegexSpecialEscapes = {'d', 'D', 's', 'S', 'w', 'W'}; while (Peek() && Peek() != ']') { if (Peek() == '\r' || Peek() == '\n') { ReportLexerError("Character class should not contain newline"); } else if (Peek() == '-') { // Handle dash; this dash could be a range expression or a normal dash. // It will further be handled in EBNFParser::ParseCharClass. tokens.push_back({TokenType::Dash, "-", "", cur_line_, cur_column_}); Consume(); } else if (Peek() == '\\' && kRegexSpecialEscapes.count(Peek(1))) { // Handle escaped characters with special function tokens.push_back( {TokenType::EscapeInCharClass, std::string(cur_, cur_ + 2), std::string(cur_ + 1, cur_ + 2), cur_line_, cur_column_} ); Consume(2); } else { // Handle normal characters auto [codepoint, len] = ParseNextUTF8OrEscaped(cur_, kRegexEscapeChars); if (codepoint == CharHandlingError::kInvalidUTF8) { ReportLexerError("Invalid UTF8 sequence"); } if (codepoint == CharHandlingError::kInvalidEscape) { ReportLexerError("Invalid escape sequence" + std::string(cur_, cur_ + 2)); } tokens.push_back( {TokenType::CharInCharClass, std::string(cur_, cur_ + len), codepoint, cur_line_, cur_column_} ); Consume(len); } } if (!Peek()) { ReportLexerError("Unterminated character class"); } tokens.push_back({TokenType::RBracket, "]", "", cur_line_, cur_column_}); Consume(); // Skip ']' return tokens; } // Parse integer EBNFLexer::Token EBNFLexer::Impl::ParseIntegerToken() { int start_line = cur_line_; int start_column = cur_column_; const char* start_pos = cur_; bool is_negative = false; if (Peek() == '-') { is_negative = true; Consume(); } else if (Peek() == '+') { Consume(); } int64_t num = 0; while (Peek() && isdigit(Peek())) { num = num * 10 + (Peek() - '0'); Consume(); if (num > kMaxIntegerInGrammar) { ReportLexerError( "Integer is too large: parsed " + std::to_string(num) + ", max allowed is " + std::to_string(kMaxIntegerInGrammar) ); } } std::string lexeme(start_pos, cur_ - start_pos); return {TokenType::IntegerLiteral, lexeme, is_negative ? -num : num, start_line, start_column}; } // Get the next token std::variant> EBNFLexer::Impl::NextToken() { ConsumeSpace(); // Skip whitespace and comments auto start_line = cur_line_; auto start_column = cur_column_; if (!Peek()) { return EBNFLexer::Token{TokenType::EndOfFile, "", "", start_line, start_column}; } // Determine token type based on current character switch (Peek()) { case '(': if (Peek(1) == '=') { Consume(2); return EBNFLexer::Token{TokenType::LookaheadLParen, "(=", "", start_line, start_column}; } else { Consume(); return EBNFLexer::Token{TokenType::LParen, "(", "", start_line, start_column}; } case ')': Consume(); return EBNFLexer::Token{TokenType::RParen, ")", "", start_line, start_column}; case '{': Consume(); return EBNFLexer::Token{TokenType::LBrace, "{", "", start_line, start_column}; case '}': Consume(); return EBNFLexer::Token{TokenType::RBrace, "}", "", start_line, start_column}; case '|': Consume(); return EBNFLexer::Token{TokenType::Pipe, "|", "", start_line, start_column}; case ',': Consume(); return EBNFLexer::Token{TokenType::Comma, ",", "", start_line, start_column}; case '*': Consume(); return EBNFLexer::Token{TokenType::Star, "*", "", start_line, start_column}; case '+': Consume(); return EBNFLexer::Token{TokenType::Plus, "+", "", start_line, start_column}; case '?': Consume(); return EBNFLexer::Token{TokenType::Question, "?", "", start_line, start_column}; case '=': Consume(); return EBNFLexer::Token{TokenType::Equal, "=", "", start_line, start_column}; case ':': if (Peek(1) == ':' && Peek(2) == '=') { Consume(3); return EBNFLexer::Token{TokenType::Assign, "::=", "", start_line, start_column}; } ReportLexerError("Unexpected character: ':'"); break; case '"': return ParseStringToken(); case '[': return ParseCharClassToken(); default: if (IsNameChar(*cur_, true)) { return ParseIdentifierOrBooleanToken(); } else if (isdigit(*cur_) || *cur_ == '-' || *cur_ == '+') { return ParseIntegerToken(); } // Unrecognized character, report error ReportLexerError("Unexpected character: " + std::string(1, *cur_)); } // Should not reach here XGRAMMAR_UNREACHABLE(); } void EBNFLexer::Impl::ConvertIdentifierToRuleName(std::vector* tokens) { for (int i = 0; i < static_cast(tokens->size()); ++i) { if (tokens->at(i).type == TokenType::Assign) { if (i == 0) { ReportLexerError( "Assign should not be the first token", tokens->at(i).line, tokens->at(i).column ); } if (tokens->at(i - 1).type != TokenType::Identifier) { ReportLexerError( "Assign should be preceded by an identifier", tokens->at(i - 1).line, tokens->at(i - 1).column ); } if (i >= 2 && tokens->at(i - 2).line == tokens->at(i - 1).line) { ReportLexerError( "The rule name should be at the beginning of the line", tokens->at(i - 1).line, tokens->at(i - 1).column ); } tokens->at(i - 1).type = TokenType::RuleName; } } } // Tokenize the entire input and return a vector of tokens std::vector EBNFLexer::Impl::Tokenize(const std::string& input) { // Reset position to the beginning input_ = input; cur_ = input_.c_str(); cur_line_ = 1; cur_column_ = 1; // Collect all tokens std::vector tokens; while (true) { auto token = NextToken(); if (auto* token_value = std::get_if(&token)) { tokens.push_back(*token_value); // Stop when we reach the end of file if (token_value->type == TokenType::EndOfFile) { break; } } else { auto vec = std::get_if>(&token); XGRAMMAR_DCHECK(vec != nullptr); tokens.insert(tokens.end(), vec->begin(), vec->end()); } } ConvertIdentifierToRuleName(&tokens); return tokens; } EBNFLexer::EBNFLexer() : pimpl_(std::make_shared()) {} std::vector EBNFLexer::Tokenize(const std::string& input) { return pimpl_->Tokenize(input); } class EBNFParser { public: /*! \brief The logic of parsing the grammar string. */ Grammar Parse( const std::vector& tokens, const std::string& root_rule_name, const int& max_nest_layer = 1000 ); private: using Rule = Grammar::Impl::Rule; using GrammarExprType = Grammar::Impl::GrammarExprType; using Token = EBNFLexer::Token; using TokenType = EBNFLexer::TokenType; // Parsing different parts of the grammar std::string ParseIdentifier(); int32_t ParseCharClass(); int32_t ParseString(); int32_t ParseRuleRef(); int32_t ParseElement(); int64_t ParseInteger(); std::pair ParseRepetitionRange(); int32_t ParseElementWithQuantifier(); int32_t ParseLookaheadAssertion(); int32_t ParseSequence(); int32_t ParseChoices(); Rule ParseRule(); // Parser for macro class MacroIR { public: struct StringNode; struct IntegerNode; struct BooleanNode; struct IdentifierNode; struct TupleNode; using Node = std::variant; using NodePtr = std::unique_ptr; struct StringNode { std::string value; }; struct IntegerNode { int64_t value; }; struct BooleanNode { bool value; }; struct IdentifierNode { std::string name; }; struct TupleNode { std::vector elements; }; struct Arguments { std::vector arguments; std::unordered_map named_arguments; }; }; MacroIR::Arguments ParseMacroArguments(); MacroIR::NodePtr ParseMacroValue(); int32_t ParseTagDispatch(); int32_t ParseTokenSet(); int32_t ParseExcludeToken(); int32_t ParseTokenTagDispatch(); // Helper functions // Helper for ParseElementWithQuantifier int32_t HandleStarQuantifier(int32_t grammar_expr_id); int32_t HandlePlusQuantifier(int32_t grammar_expr_id); int32_t HandleQuestionQuantifier(int32_t grammar_expr_id); // When parsing, we first find the names of all rules, and build the mapping from name to rule id. void InitRuleNames(); // Consume a token and advance to the next void Consume(int cnt = 1); // Peek at the current token with optional offset const Token& Peek(int delta = 0) const; // Consume token if it matches expected type, otherwise report error void PeekAndConsume(TokenType type, const std::string& message); // Report a parsing error with the given message [[noreturn]] void ReportParseError(const std::string& msg, int delta_element = 0); // The grammar builder GrammarBuilder builder_; // The current token pointer const Token* current_token_ = nullptr; // Tokens from lexer std::vector tokens_; // The current rule name. Help to generate a name for a new rule. std::string cur_rule_name_; // The name of the root rule std::string root_rule_name_; int nest_layer_guard_ = 0; int max_nest_layer_ = 1000; // Max nest layer of the grammar static const std::unordered_map> kMacroFunctions; }; const std::unordered_map> EBNFParser::kMacroFunctions = { {"TagDispatch", [](EBNFParser* parser) { return parser->ParseTagDispatch(); }}, {"Token", [](EBNFParser* parser) { return parser->ParseTokenSet(); }}, {"ExcludeToken", [](EBNFParser* parser) { return parser->ParseExcludeToken(); }}, {"TokenTagDispatch", [](EBNFParser* parser) { return parser->ParseTokenTagDispatch(); }}, }; const EBNFParser::Token& EBNFParser::Peek(int delta) const { return *(current_token_ + delta); } void EBNFParser::Consume(int cnt) { current_token_ += cnt; } void EBNFParser::PeekAndConsume(TokenType type, const std::string& message) { if (Peek().type != type) { ReportParseError(message); } Consume(); } void EBNFParser::ReportParseError(const std::string& msg, int delta_element) { XGRAMMAR_DCHECK(current_token_ + delta_element < tokens_.data() + tokens_.size()); int line_to_print = Peek(delta_element).line; int column_to_print = Peek(delta_element).column; XGRAMMAR_LOG(FATAL) << "EBNF parser error at line " + std::to_string(line_to_print) + ", column " + std::to_string(column_to_print) + ": " + msg; XGRAMMAR_UNREACHABLE(); } std::string EBNFParser::ParseIdentifier() { if (Peek().type != TokenType::Identifier) { ReportParseError("Expect identifier"); } std::string identifier = std::any_cast(Peek().value); Consume(); return identifier; } int32_t EBNFParser::ParseCharClass() { PeekAndConsume(TokenType::LBracket, "Expect [ in character class"); std::vector elements; bool is_negated = false; if (Peek().type == TokenType::Caret) { is_negated = true; Consume(); } while (Peek().type != TokenType::RBracket && Peek().type != TokenType::EndOfFile) { if (Peek().type == TokenType::EscapeInCharClass) { ReportParseError("Character class escape is not supported yet in EBNF"); } TCodepoint codepoint; if (Peek().type == TokenType::CharInCharClass) { codepoint = std::any_cast(Peek().value); } else if (Peek().type == TokenType::Dash) { codepoint = static_cast(static_cast('-')); } else { ReportParseError("Unexpected character in character class: " + Peek().lexeme); } Consume(); if (Peek().type == TokenType::Dash && (Peek(1).type == TokenType::CharInCharClass || Peek(1).type == TokenType::Dash)) { // Range expression TCodepoint codepoint2; if (Peek(1).type == TokenType::CharInCharClass) { codepoint2 = std::any_cast(Peek(1).value); } else { XGRAMMAR_DCHECK(Peek(1).type == TokenType::Dash); codepoint2 = static_cast(static_cast('-')); } if (codepoint > codepoint2) { ReportParseError("Invalid character class: lower bound is larger than upper bound", -1); } elements.push_back({codepoint, codepoint2}); Consume(2); } else { // Single character elements.push_back({codepoint, codepoint}); } } PeekAndConsume(TokenType::RBracket, "Expect ] in character class"); return builder_.AddCharacterClass(elements, is_negated); } int32_t EBNFParser::ParseString() { if (Peek().type != TokenType::StringLiteral) { ReportParseError("Expect string literal"); } std::string str_value = std::any_cast(Peek().value); Consume(); if (str_value.empty()) { return builder_.AddEmptyStr(); } return builder_.AddByteString(str_value); } int32_t EBNFParser::ParseRuleRef() { std::string name = ParseIdentifier(); auto rule_id = builder_.GetRuleId(name); if (rule_id == -1) { ReportParseError("Rule \"" + name + "\" is not defined", -1); } return builder_.AddRuleRef(rule_id); } int32_t EBNFParser::ParseElement() { if (Peek().type == TokenType::LParen) { nest_layer_guard_++; if (nest_layer_guard_ > max_nest_layer_) { ReportParseError("Nest layer exceeded the maximum limit", -1); } Consume(); if (Peek().type == TokenType::RParen) { // Special case: ( ) Consume(); nest_layer_guard_--; return builder_.AddEmptyStr(); } auto grammar_expr_id = ParseChoices(); PeekAndConsume(TokenType::RParen, "Expect )"); nest_layer_guard_--; return grammar_expr_id; } else if (Peek().type == TokenType::LBracket) { return ParseCharClass(); } else if (Peek().type == TokenType::StringLiteral) { return ParseString(); } else if (Peek().type == TokenType::Identifier) { auto id = std::any_cast(Peek().value); if (kMacroFunctions.count(id)) { return kMacroFunctions.at(id)(this); } else { return ParseRuleRef(); } } else { ReportParseError("Expect element, but got " + Peek().lexeme); } } int64_t EBNFParser::ParseInteger() { if (Peek().type != TokenType::IntegerLiteral) { ReportParseError("Expect integer, but got " + Peek().lexeme); } int64_t num = std::any_cast(Peek().value); Consume(); return num; } std::pair EBNFParser::ParseRepetitionRange() { PeekAndConsume(TokenType::LBrace, "Expect {"); int64_t lower = ParseInteger(); if (lower < 0) { ReportParseError("Lower bound cannot be negative", -1); } if (Peek().type == TokenType::Comma) { Consume(); if (Peek().type == TokenType::RBrace) { Consume(); return {lower, -1}; } // The grammar printer emits {n, -1} for unbounded upper bounds, and // '-' is a valid identifier-start char (IsNameChar), so the lexer // produces Identifier("-1") rather than IntegerLiteral. Accept it // as equivalent to {n,}. if (Peek().type == TokenType::Identifier && Peek().lexeme == "-1") { Consume(); PeekAndConsume(TokenType::RBrace, "Expect }"); return {lower, -1}; } int64_t upper = ParseInteger(); if (upper < lower) { ReportParseError( "Lower bound is larger than upper bound: " + std::to_string(lower) + " > " + std::to_string(upper), -1 ); } PeekAndConsume(TokenType::RBrace, "Expect }"); return {lower, upper}; } else if (Peek().type == TokenType::RBrace) { Consume(); return {lower, lower}; } ReportParseError("Expect ',' or '}' in repetition range"); } int32_t EBNFParser::HandleStarQuantifier(int32_t grammar_expr_id) { Grammar::Impl::GrammarExpr grammar_expr = builder_.GetGrammarExpr(grammar_expr_id); if (grammar_expr.type == GrammarBuilder::GrammarExprType::kCharacterClass) { // We have special handling for character class star, e.g. [a-z]* grammar_expr.type = GrammarBuilder::GrammarExprType::kCharacterClassStar; // Copy grammar expr because the grammar may change during insertion, and grammar_expr is in the // grammar, so it may become invalid std::vector grammar_expr_data(grammar_expr.begin(), grammar_expr.end()); return builder_.AddGrammarExpr( {grammar_expr.type, grammar_expr_data.data(), grammar_expr.data_len} ); } else { // For other star quantifiers, we transform it into a rule: // a* --> rule ::= a rule | "" auto new_rule_name = builder_.GetNewRuleName(cur_rule_name_); auto new_rule_id = builder_.AddEmptyRule(new_rule_name); auto ref_to_new_rule = builder_.AddRuleRef(new_rule_id); auto new_grammar_expr_id = builder_.AddChoices( {builder_.AddEmptyStr(), builder_.AddSequence({grammar_expr_id, ref_to_new_rule})} ); builder_.UpdateRuleBody(new_rule_id, new_grammar_expr_id); // Return the reference to the new rule return builder_.AddRuleRef(new_rule_id); } } int32_t EBNFParser::HandlePlusQuantifier(int32_t grammar_expr_id) { // a+ --> rule ::= a rule | a auto new_rule_name = builder_.GetNewRuleName(cur_rule_name_); auto new_rule_id = builder_.AddEmptyRule(new_rule_name); auto ref_to_new_rule = builder_.AddRuleRef(new_rule_id); auto new_grammar_expr_id = builder_.AddChoices( {builder_.AddSequence({grammar_expr_id, ref_to_new_rule}), grammar_expr_id} ); builder_.UpdateRuleBody(new_rule_id, new_grammar_expr_id); // Return the reference to the new rule return builder_.AddRuleRef(new_rule_id); } int32_t EBNFParser::HandleQuestionQuantifier(int32_t grammar_expr_id) { // a? --> rule ::= a | empty auto new_rule_name = builder_.GetNewRuleName(cur_rule_name_); auto new_grammar_expr_id = builder_.AddChoices({builder_.AddEmptyStr(), grammar_expr_id}); auto new_rule_id = builder_.AddRule({new_rule_name, new_grammar_expr_id}); return builder_.AddRuleRef(new_rule_id); } int32_t EBNFParser::ParseElementWithQuantifier() { int32_t grammar_expr_id = ParseElement(); if (Peek().type == TokenType::Star) { Consume(); return HandleStarQuantifier(grammar_expr_id); } else if (Peek().type == TokenType::Plus) { Consume(); return HandlePlusQuantifier(grammar_expr_id); } else if (Peek().type == TokenType::Question) { Consume(); return HandleQuestionQuantifier(grammar_expr_id); } else if (Peek().type == TokenType::LBrace) { auto [lower, upper] = ParseRepetitionRange(); return builder_.AddRepeatFromExpr( cur_rule_name_, grammar_expr_id, static_cast(lower), upper == -1 ? -1 : static_cast(upper) ); } return grammar_expr_id; } int32_t EBNFParser::ParseSequence() { std::vector elements; do { elements.push_back(ParseElementWithQuantifier()); } while (Peek().type != TokenType::Pipe && Peek().type != TokenType::RParen && Peek().type != TokenType::LookaheadLParen && Peek().type != TokenType::RuleName && Peek().type != TokenType::EndOfFile); return builder_.AddSequence(elements); } int32_t EBNFParser::ParseChoices() { std::vector choices; choices.push_back(ParseSequence()); while (Peek().type == TokenType::Pipe) { Consume(); choices.push_back(ParseSequence()); } return builder_.AddChoices(choices); } // Parse macro arguments and return a MacroIR::Arguments structure EBNFParser::MacroIR::Arguments EBNFParser::ParseMacroArguments() { MacroIR::Arguments args; PeekAndConsume(TokenType::LParen, "Expect ( after macro function name"); // Parse arguments if (Peek().type != TokenType::RParen) { while (true) { // Check if it's a named argument (identifier = value) if (Peek().type == TokenType::Identifier && Peek(1).type == TokenType::Equal) { std::string name = std::any_cast(Peek().value); Consume(); // Consume identifier Consume(); // Consume = // Parse the value args.named_arguments[name] = ParseMacroValue(); } else { // Regular positional argument args.arguments.push_back(ParseMacroValue()); } // Check for comma or end of arguments if (Peek().type == TokenType::Comma) { Consume(); } else if (Peek().type == TokenType::RParen) { break; } else { ReportParseError("Expect , or ) in macro arguments"); } } } PeekAndConsume(TokenType::RParen, "Expect ) after macro arguments"); return args; } // Parse a single macro value (string, integer, boolean, or tuple) EBNFParser::MacroIR::NodePtr EBNFParser::ParseMacroValue() { if (Peek().type == TokenType::StringLiteral) { // String value std::string value = std::any_cast(Peek().value); Consume(); return std::make_unique(MacroIR::StringNode{value}); } else if (Peek().type == TokenType::IntegerLiteral) { // Integer value int64_t value = std::any_cast(Peek().value); Consume(); return std::make_unique(MacroIR::IntegerNode{value}); } else if (Peek().type == TokenType::BooleanLiteral) { // Boolean value bool value = std::any_cast(Peek().value); Consume(); return std::make_unique(MacroIR::BooleanNode{value}); } else if (Peek().type == TokenType::Identifier) { // Identifier value std::string name = std::any_cast(Peek().value); Consume(); return std::make_unique(MacroIR::IdentifierNode{name}); } else if (Peek().type == TokenType::LParen) { // Tuple value Consume(); // Consume ( MacroIR::TupleNode tuple; // Parse tuple elements (supports trailing comma) if (Peek().type != TokenType::RParen) { while (true) { tuple.elements.push_back(ParseMacroValue()); if (Peek().type == TokenType::Comma) { Consume(); if (Peek().type == TokenType::RParen) { break; } } else if (Peek().type == TokenType::RParen) { break; } else { ReportParseError("Expect , or ) in tuple"); } } } Consume(); // Consume ) return std::make_unique(std::move(tuple)); } else { ReportParseError("Expect string, integer, boolean, or tuple in macro argument"); } } int32_t EBNFParser::ParseTagDispatch() { Consume(); // Consume TagDispatch operator auto start = current_token_; auto args = ParseMacroArguments(); auto delta_element = start - current_token_; // Used to report parse errors Grammar::Impl::TagDispatch tag_dispatch; static const std::unordered_set kValidNamedArgs = { "loop_after_dispatch", "excludes" }; for (const auto& [name, _] : args.named_arguments) { if (kValidNamedArgs.count(name) == 0) { ReportParseError("Unknown named argument for TagDispatch: " + name, delta_element); } } // Positional parameters: ("tag_string", rule_name) — string triggers only for (const auto& arg : args.arguments) { auto tuple_node = std::get_if(arg.get()); if (tuple_node == nullptr) { ReportParseError("Each tag dispatch element must be a tuple", delta_element); } if (tuple_node->elements.size() != 2) { ReportParseError("Each tag dispatch element must be a pair (tag, rule)", delta_element); } auto tag_str_node = std::get_if(tuple_node->elements[0].get()); if (tag_str_node == nullptr || tag_str_node->value.empty()) { ReportParseError("Tag must be a non-empty string literal", delta_element); } auto rule_name_node = std::get_if(tuple_node->elements[1].get()); if (rule_name_node == nullptr) { ReportParseError("Rule reference must be an identifier", delta_element); } auto rule_id = builder_.GetRuleId(rule_name_node->name); if (rule_id == -1) { ReportParseError("Rule \"" + rule_name_node->name + "\" is not defined", delta_element); } tag_dispatch.tag_rule_pairs.push_back({tag_str_node->value, rule_id}); } // loop_after_dispatch tag_dispatch.loop_after_dispatch = true; if (auto it = args.named_arguments.find("loop_after_dispatch"); it != args.named_arguments.end()) { auto bool_node = std::get_if(it->second.get()); if (bool_node == nullptr) { ReportParseError("loop_after_dispatch must be a boolean literal", delta_element); } tag_dispatch.loop_after_dispatch = bool_node->value; } // excludes — string only if (auto it = args.named_arguments.find("excludes"); it != args.named_arguments.end()) { auto tuple_node = std::get_if(it->second.get()); if (tuple_node == nullptr) { ReportParseError("excludes must be a tuple", delta_element); } for (const auto& element : tuple_node->elements) { auto str_node = std::get_if(element.get()); if (str_node == nullptr || str_node->value.empty()) { ReportParseError("Exclude must be a non-empty string literal", delta_element); } tag_dispatch.excludes.push_back(str_node->value); } } // Well-formedness checks: string excludes vs string triggers for (const auto& excl_str : tag_dispatch.excludes) { for (const auto& [trigger_str, _] : tag_dispatch.tag_rule_pairs) { if (trigger_str.rfind(excl_str, 0) == 0) { ReportParseError( "Exclude string must not be a prefix of trigger string: " + excl_str, delta_element ); } } } return builder_.AddTagDispatch(tag_dispatch); } int32_t EBNFParser::ParseTokenSet() { Consume(); // Consume Token identifier auto start = current_token_; auto args = ParseMacroArguments(); auto delta_element = start - current_token_; if (!args.named_arguments.empty()) { ReportParseError("Token() does not accept named arguments", delta_element); } if (args.arguments.empty()) { ReportParseError("Token() requires at least one integer argument", delta_element); } std::vector token_ids; for (const auto& arg : args.arguments) { auto int_node = std::get_if(arg.get()); if (int_node == nullptr || int_node->value < 0) { ReportParseError("Token() arguments must be non-negative integers", delta_element); } token_ids.push_back(static_cast(int_node->value)); } std::sort(token_ids.begin(), token_ids.end()); token_ids.erase(std::unique(token_ids.begin(), token_ids.end()), token_ids.end()); return builder_.AddTokenSet(token_ids); } int32_t EBNFParser::ParseExcludeToken() { Consume(); auto start = current_token_; auto args = ParseMacroArguments(); auto delta_element = start - current_token_; if (!args.named_arguments.empty()) { ReportParseError("ExcludeToken() does not accept named arguments", delta_element); } if (args.arguments.empty()) { ReportParseError("ExcludeToken() requires at least one integer argument", delta_element); } std::vector token_ids; for (const auto& arg : args.arguments) { auto int_node = std::get_if(arg.get()); if (int_node == nullptr || int_node->value < 0) { ReportParseError("ExcludeToken() arguments must be non-negative integers", delta_element); } token_ids.push_back(static_cast(int_node->value)); } std::sort(token_ids.begin(), token_ids.end()); token_ids.erase(std::unique(token_ids.begin(), token_ids.end()), token_ids.end()); return builder_.AddExcludeTokenSet(token_ids); } int32_t EBNFParser::ParseTokenTagDispatch() { Consume(); auto start = current_token_; auto args = ParseMacroArguments(); auto delta_element = start - current_token_; Grammar::Impl::TokenTagDispatch ttd; static const std::unordered_set kValidNamedArgs = { "loop_after_dispatch", "excludes" }; for (const auto& [name, _] : args.named_arguments) { if (kValidNamedArgs.count(name) == 0) { ReportParseError("Unknown named argument for TokenTagDispatch: " + name, delta_element); } } for (const auto& arg : args.arguments) { auto tuple_node = std::get_if(arg.get()); if (tuple_node == nullptr || tuple_node->elements.size() != 2) { ReportParseError( "Each TokenTagDispatch element must be a pair (token_id, rule)", delta_element ); } auto id_node = std::get_if(tuple_node->elements[0].get()); if (id_node == nullptr || id_node->value < 0) { ReportParseError("Token trigger ID must be a non-negative integer", delta_element); } auto rule_node = std::get_if(tuple_node->elements[1].get()); if (rule_node == nullptr) { ReportParseError("Rule reference must be an identifier", delta_element); } auto rule_id = builder_.GetRuleId(rule_node->name); if (rule_id == -1) { ReportParseError("Rule \"" + rule_node->name + "\" is not defined", delta_element); } ttd.trigger_rule_pairs.push_back({static_cast(id_node->value), rule_id}); } ttd.loop_after_dispatch = true; if (auto it = args.named_arguments.find("loop_after_dispatch"); it != args.named_arguments.end()) { auto bool_node = std::get_if(it->second.get()); if (bool_node == nullptr) { ReportParseError("loop_after_dispatch must be a boolean", delta_element); } ttd.loop_after_dispatch = bool_node->value; } if (auto it = args.named_arguments.find("excludes"); it != args.named_arguments.end()) { auto tuple_node = std::get_if(it->second.get()); if (tuple_node == nullptr) { ReportParseError("excludes must be a tuple", delta_element); } for (const auto& element : tuple_node->elements) { auto int_node = std::get_if(element.get()); if (int_node == nullptr || int_node->value < 0) { ReportParseError("Exclude token ID must be a non-negative integer", delta_element); } ttd.excludes.push_back(static_cast(int_node->value)); } } for (auto excl_id : ttd.excludes) { for (const auto& [tid, _] : ttd.trigger_rule_pairs) { if (tid == excl_id) { ReportParseError( "Token trigger ID " + std::to_string(tid) + " must not overlap with exclude token ID", delta_element ); } } } return builder_.AddTokenTagDispatch(ttd); } int32_t EBNFParser::ParseLookaheadAssertion() { PeekAndConsume(TokenType::LookaheadLParen, "Expect (= in lookahead assertion"); auto result = ParseChoices(); PeekAndConsume(TokenType::RParen, "Expect )"); return result; } EBNFParser::Rule EBNFParser::ParseRule() { if (Peek().type != TokenType::RuleName) { ReportParseError("Expect rule name"); } cur_rule_name_ = std::any_cast(Peek().value); Consume(); PeekAndConsume(TokenType::Assign, "Expect ::="); auto body_id = ParseChoices(); int32_t lookahead_id = -1; if (Peek().type == TokenType::LookaheadLParen) { lookahead_id = ParseLookaheadAssertion(); } return {cur_rule_name_, body_id, lookahead_id}; } void EBNFParser::InitRuleNames() { int delta_element = 0; for (auto& token : tokens_) { if (token.type == TokenType::RuleName) { auto name = std::any_cast(token.value); if (builder_.GetRuleId(name) != -1) { ReportParseError("Rule \"" + name + "\" is defined multiple times", delta_element); } builder_.AddEmptyRule(name); } ++delta_element; } if (builder_.GetRuleId(root_rule_name_) == -1) { ReportParseError("The root rule with name \"" + root_rule_name_ + "\" is not found", 0); } } Grammar EBNFParser::Parse( const std::vector& tokens, const std::string& root_rule_name, const int& max_nest_layer ) { max_nest_layer_ = max_nest_layer; nest_layer_guard_ = 0; tokens_ = tokens; current_token_ = tokens_.data(); root_rule_name_ = root_rule_name; // First collect rule names InitRuleNames(); // Then parse all the rules while (Peek().type != TokenType::EndOfFile) { auto new_rule = ParseRule(); builder_.UpdateRuleBody(new_rule.name, new_rule.body_expr_id); builder_.UpdateLookaheadAssertion(new_rule.name, new_rule.lookahead_assertion_id); } return builder_.Get(root_rule_name); } Grammar ParseEBNF(const std::string& ebnf_string, const std::string& root_rule_name) { EBNFLexer lexer; auto tokens = lexer.Tokenize(ebnf_string); EBNFParser parser; return parser.Parse(std::move(tokens), root_rule_name); } } // namespace xgrammar xgrammar-0.2.3/cpp/grammar_parser.h000066400000000000000000000052011521764210300172660ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar_parser.h * \brief The header for the parser of BNF/EBNF grammar into BNF AST. */ #ifndef XGRAMMAR_GRAMMAR_PARSER_H_ #define XGRAMMAR_GRAMMAR_PARSER_H_ #include #include namespace xgrammar { class EBNFLexer { public: // Token types enum class TokenType { RuleName, // the name of a rule definition, e.g.: root, rule1 Identifier, // reference to a rule, or a Macro name, e.g.: root, rule1, TagDispatch StringLiteral, // e.g.: "tag1", "hello" BooleanLiteral, // true, false IntegerLiteral, // 123 LParen, // ( RParen, // ) LBrace, // { RBrace, // } Pipe, // | Comma, // , EndOfFile, // End of file // Symbols and quantifiers Assign, // ::= Equal, // = Star, // * Plus, // + Question, // ? // Character class LBracket, // [ RBracket, // ] Dash, // - Caret, // ^ CharInCharClass, // a character in a character class, e.g. a and z in [a-z]; escaped chars // with no special meaning are also included, e.g. . in [a\.z] EscapeInCharClass, // Escaped sequence with special function, e.g. \S in [\S] // Special structures LookaheadLParen, // (= }; // Token structure struct Token { TokenType type; std::string lexeme; // original text std::any value; // The processed value. Can be a int for integer literal, a string for string // literal, etc. int line; int column; }; EBNFLexer(); std::vector Tokenize(const std::string& input); XGRAMMAR_DEFINE_PIMPL_METHODS(EBNFLexer); }; /*! * \brief This class parses a BNF/EBNF grammar string into an BNF abstract syntax tree (AST). * \details This function accepts the EBNF notation defined in the W3C XML Specification * (https://www.w3.org/TR/xml/#sec-notation), which is a popular standard, with the following * changes: * - Using # as comment mark instead of C-style comments * - Accept C-style unicode escape sequence \u01AB, \U000001AB, \xAB instead of #x0123 * - Rule A-B (match A and not match B) is not supported yet * * See tests/python/serve/json.ebnf for an example. * \param ebnf_string The grammar string. * \param root_rule_name The name of the root rule. Default is "root". * \return The parsed grammar. */ Grammar ParseEBNF(const std::string& ebnf_string, const std::string& root_rule_name = "root"); } // namespace xgrammar #endif // XGRAMMAR_GRAMMAR_PARSER_H_ xgrammar-0.2.3/cpp/grammar_printer.cc000066400000000000000000000152541521764210300176240ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar_printer.cc */ #include "grammar_printer.h" #include #include "support/encoding.h" namespace xgrammar { std::string GrammarPrinter::PrintRule(const Rule& rule) { std::string res = rule.name + " ::= " + PrintGrammarExpr(rule.body_expr_id); if (rule.lookahead_assertion_id != -1) { res += " (=" + PrintGrammarExpr(rule.lookahead_assertion_id) + ")"; } return res; } std::string GrammarPrinter::PrintRule(int32_t rule_id) { return PrintRule(grammar_->GetRule(rule_id)); } std::string GrammarPrinter::PrintGrammarExpr(const GrammarExpr& grammar_expr) { std::string result; switch (grammar_expr.type) { case GrammarExprType::kByteString: return PrintByteString(grammar_expr); case GrammarExprType::kCharacterClass: return PrintCharacterClass(grammar_expr); case GrammarExprType::kCharacterClassStar: return PrintCharacterClassStar(grammar_expr); case GrammarExprType::kEmptyStr: return PrintEmptyStr(grammar_expr); case GrammarExprType::kRuleRef: return PrintRuleRef(grammar_expr); case GrammarExprType::kSequence: return PrintSequence(grammar_expr); case GrammarExprType::kChoices: return PrintChoices(grammar_expr); case GrammarExprType::kTagDispatch: return PrintTagDispatch(grammar_expr); case GrammarExprType::kRepeat: return PrintRepeat(grammar_expr); case GrammarExprType::kToken: return PrintToken(grammar_expr); case GrammarExprType::kExcludeToken: return PrintExcludeToken(grammar_expr); case GrammarExprType::kTokenTagDispatch: return PrintTokenTagDispatch(grammar_expr); default: XGRAMMAR_LOG(FATAL) << "Unexpected GrammarExpr type: " << static_cast(grammar_expr.type); XGRAMMAR_UNREACHABLE(); } } std::string GrammarPrinter::PrintGrammarExpr(int32_t grammar_expr_id) { return PrintGrammarExpr(grammar_->GetGrammarExpr(grammar_expr_id)); } std::string GrammarPrinter::PrintByteString(const GrammarExpr& grammar_expr) { std::string internal_str; internal_str.reserve(grammar_expr.data_len); for (int i = 0; i < grammar_expr.data_len; ++i) { internal_str += static_cast(grammar_expr[i]); } return "\"" + EscapeString(internal_str) + "\""; } std::string GrammarPrinter::PrintCharacterClass(const GrammarExpr& grammar_expr) { static const std::unordered_map kCustomEscapeMap = { {'-', "\\-"}, {']', "\\]"} }; std::string result = "["; bool is_negative = static_cast(grammar_expr[0]); if (is_negative) { result += "^"; } for (auto i = 1; i < grammar_expr.data_len; i += 2) { result += EscapeString(grammar_expr[i], kCustomEscapeMap); if (grammar_expr[i] == grammar_expr[i + 1]) { continue; } result += "-"; result += EscapeString(grammar_expr[i + 1], kCustomEscapeMap); } result += "]"; return result; } std::string GrammarPrinter::PrintCharacterClassStar(const GrammarExpr& grammar_expr) { return PrintCharacterClass(grammar_expr) + "*"; } std::string GrammarPrinter::PrintEmptyStr(const GrammarExpr& grammar_expr) { return "\"\""; } std::string GrammarPrinter::PrintRuleRef(const GrammarExpr& grammar_expr) { return grammar_->GetRule(grammar_expr[0]).name; } std::string GrammarPrinter::PrintSequence(const GrammarExpr& grammar_expr) { std::string result; result += "("; for (int i = 0; i < grammar_expr.data_len; ++i) { result += PrintGrammarExpr(grammar_expr[i]); if (i + 1 != grammar_expr.data_len) { result += " "; } } result += ")"; return result; } std::string GrammarPrinter::PrintChoices(const GrammarExpr& grammar_expr) { std::string result; result += "("; for (int i = 0; i < grammar_expr.data_len; ++i) { result += PrintGrammarExpr(grammar_expr[i]); if (i + 1 != grammar_expr.data_len) { result += " | "; } } result += ")"; return result; } std::string GrammarPrinter::PrintString(const std::string& str) { return "\"" + EscapeString(str) + "\""; } std::string GrammarPrinter::PrintBoolean(bool value) { return value ? "true" : "false"; } std::string GrammarPrinter::PrintTagDispatch(const GrammarExpr& grammar_expr) { auto tag_dispatch = grammar_->GetTagDispatch(grammar_expr); std::string result = "TagDispatch(\n"; std::string indent = " "; for (const auto& [trigger, rule_id] : tag_dispatch.tag_rule_pairs) { result += indent + "(" + PrintString(trigger) + ", " + grammar_->GetRule(rule_id).name + "),\n"; } result += indent + "loop_after_dispatch=" + PrintBoolean(tag_dispatch.loop_after_dispatch) + ",\n"; result += indent + "excludes=("; for (int i = 0; i < static_cast(tag_dispatch.excludes.size()); ++i) { if (i > 0) result += ", "; result += PrintString(tag_dispatch.excludes[i]); } result += ")\n)"; return result; } std::string GrammarPrinter::PrintRepeat(const GrammarExpr& grammar_expr) { int32_t lower_bound = grammar_expr[1]; int32_t upper_bound = grammar_expr[2]; std::string result = grammar_->GetRule(grammar_expr[0]).name + "{"; result += std::to_string(lower_bound); result += ", "; result += std::to_string(upper_bound); result += "}"; return result; } std::string GrammarPrinter::PrintToken(const GrammarExpr& grammar_expr) { std::string result = "Token("; for (int i = 0; i < grammar_expr.data_len; ++i) { if (i > 0) result += ", "; result += std::to_string(grammar_expr[i]); } result += ")"; return result; } std::string GrammarPrinter::PrintExcludeToken(const GrammarExpr& grammar_expr) { std::string result = "ExcludeToken("; for (int i = 0; i < grammar_expr.data_len; ++i) { if (i > 0) result += ", "; result += std::to_string(grammar_expr[i]); } result += ")"; return result; } std::string GrammarPrinter::PrintTokenTagDispatch(const GrammarExpr& grammar_expr) { auto ttd = grammar_->GetTokenTagDispatch(grammar_expr); std::string result = "TokenTagDispatch(\n"; std::string indent = " "; for (const auto& [token_id, rule_id] : ttd.trigger_rule_pairs) { result += indent + "(" + std::to_string(token_id) + ", " + grammar_->GetRule(rule_id).name + "),\n"; } result += indent + "loop_after_dispatch=" + PrintBoolean(ttd.loop_after_dispatch) + ",\n"; result += indent + "excludes=("; for (int i = 0; i < static_cast(ttd.excludes.size()); ++i) { if (i > 0) result += ", "; result += std::to_string(ttd.excludes[i]); } result += ")\n)"; return result; } std::string GrammarPrinter::ToString() { std::string result; int num_rules = grammar_->NumRules(); for (auto i = 0; i < num_rules; ++i) { result += PrintRule(grammar_->GetRule(i)) + "\n"; } return result; } } // namespace xgrammar xgrammar-0.2.3/cpp/grammar_printer.h000066400000000000000000000055341521764210300174660ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar_printer.h * \brief The header for printing the AST of a BNF grammar. */ #ifndef XGRAMMAR_GRAMMAR_PRINTER_H_ #define XGRAMMAR_GRAMMAR_PRINTER_H_ #include #include #include "grammar_impl.h" namespace xgrammar { /*! * \brief Prints the BNF AST with standard BNF format. */ class GrammarPrinter { private: using Rule = Grammar::Impl::Rule; using GrammarExprType = Grammar::Impl::GrammarExprType; using GrammarExpr = Grammar::Impl::GrammarExpr; public: /*! * \brief Constructor. * \param grammar The grammar to print. */ explicit GrammarPrinter(const Grammar& grammar) : grammar_(grammar) {} /*! \brief Print the complete grammar. */ std::string ToString(); /*! \brief Print a rule. */ std::string PrintRule(const Rule& rule); /*! \brief Print a rule corresponding to the given id. */ std::string PrintRule(int32_t rule_id); /*! \brief Print a GrammarExpr. */ std::string PrintGrammarExpr(const GrammarExpr& grammar_expr); /*! \brief Print a GrammarExpr corresponding to the given id. */ std::string PrintGrammarExpr(int32_t grammar_expr_id); private: /*! \brief Print a GrammarExpr for byte string. */ std::string PrintByteString(const GrammarExpr& grammar_expr); /*! \brief Print a GrammarExpr for character class. */ std::string PrintCharacterClass(const GrammarExpr& grammar_expr); /*! \brief Print a GrammarExpr for a star quantifier of a character class. */ std::string PrintCharacterClassStar(const GrammarExpr& grammar_expr); /*! \brief Print a GrammarExpr for empty string. */ std::string PrintEmptyStr(const GrammarExpr& grammar_expr); /*! \brief Print a GrammarExpr for rule reference. */ std::string PrintRuleRef(const GrammarExpr& grammar_expr); /*! \brief Print a GrammarExpr for grammar_expr sequence. */ std::string PrintSequence(const GrammarExpr& grammar_expr); /*! \brief Print a GrammarExpr for grammar_expr choices. */ std::string PrintChoices(const GrammarExpr& grammar_expr); /*! \brief Print a GrammarExpr for tag dispatch. */ std::string PrintTagDispatch(const GrammarExpr& grammar_expr); /*! \brief Print a GrammarExpr for repeat. */ std::string PrintRepeat(const GrammarExpr& grammar_expr); /*! \brief Print a GrammarExpr for token. */ std::string PrintToken(const GrammarExpr& grammar_expr); /*! \brief Print a GrammarExpr for exclude token. */ std::string PrintExcludeToken(const GrammarExpr& grammar_expr); /*! \brief Print a GrammarExpr for token tag dispatch. */ std::string PrintTokenTagDispatch(const GrammarExpr& grammar_expr); /*! \brief Print a string. */ std::string PrintString(const std::string& str); /*! \brief Print a boolean. */ std::string PrintBoolean(bool value); Grammar grammar_; }; } // namespace xgrammar #endif // XGRAMMAR_GRAMMAR_PRINTER_H_ xgrammar-0.2.3/cpp/json_schema_converter.cc000066400000000000000000004006501521764210300210110ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/json_schema_converter.cc * \brief Implementation of JSONSchemaConverter and related utilities. */ #include "json_schema_converter.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "json_schema_converter_ext.h" #include "regex_converter.h" #include "support/logging.h" #include "support/utils.h" namespace xgrammar { // ==================== Spec ToString implementations ==================== std::string IntegerSpec::ToString() const { return "IntegerSpec{minimum=" + (minimum.has_value() ? std::to_string(*minimum) : "null") + ", maximum=" + (maximum.has_value() ? std::to_string(*maximum) : "null") + ", exclusive_minimum=" + (exclusive_minimum.has_value() ? std::to_string(*exclusive_minimum) : "null") + ", exclusive_maximum=" + (exclusive_maximum.has_value() ? std::to_string(*exclusive_maximum) : "null") + "}"; } std::string NumberSpec::ToString() const { return "NumberSpec{minimum=" + (minimum.has_value() ? std::to_string(*minimum) : "null") + ", maximum=" + (maximum.has_value() ? std::to_string(*maximum) : "null") + ", exclusive_minimum=" + (exclusive_minimum.has_value() ? std::to_string(*exclusive_minimum) : "null") + ", exclusive_maximum=" + (exclusive_maximum.has_value() ? std::to_string(*exclusive_maximum) : "null") + "}"; } std::string StringSpec::ToString() const { return "StringSpec{pattern=" + (pattern.has_value() ? "\"" + *pattern + "\"" : "null") + ", format=" + (format.has_value() ? "\"" + *format + "\"" : "null") + ", min_length=" + std::to_string(min_length) + ", max_length=" + std::to_string(max_length) + "}"; } std::string BooleanSpec::ToString() const { return "BooleanSpec{}"; } std::string NullSpec::ToString() const { return "NullSpec{}"; } std::string AnySpec::ToString() const { return "AnySpec{}"; } std::string ArraySpec::ToString() const { return "ArraySpec{prefix_items.size()=" + std::to_string(prefix_items.size()) + ", allow_additional_items=" + (allow_additional_items ? "true" : "false") + ", additional_items=" + (additional_items ? "SchemaSpec" : "null") + ", min_items=" + std::to_string(min_items) + ", max_items=" + std::to_string(max_items) + "}"; } std::string ObjectSpec::ToString() const { std::string s = "ObjectSpec{properties.size()=" + std::to_string(properties.size()) + ", properties=["; for (size_t i = 0; i < properties.size(); ++i) { if (i != 0) s += ", "; s += properties[i].name; } s += "], pattern_properties.size()=" + std::to_string(pattern_properties.size()) + ", required=["; bool first = true; for (const auto& r : required) { if (!first) s += ", "; s += r; first = false; } s += std::string("], allow_additional_properties=") + (allow_additional_properties ? "true" : "false") + ", additional_properties_schema=" + (additional_properties_schema ? "SchemaSpec" : "null") + ", allow_unevaluated_properties=" + (allow_unevaluated_properties ? "true" : "false") + ", unevaluated_properties_schema=" + (unevaluated_properties_schema ? "SchemaSpec" : "null") + ", property_names=" + (property_names ? "SchemaSpec" : "null") + ", min_properties=" + std::to_string(min_properties) + ", max_properties=" + std::to_string(max_properties) + "}"; return s; } std::string ConstSpec::ToString() const { return "ConstSpec{json_value=\"" + json_value + "\"}"; } std::string EnumSpec::ToString() const { std::string s = "EnumSpec{json_values.size()=" + std::to_string(json_values.size()) + ", json_values=["; for (size_t i = 0; i < json_values.size(); ++i) { if (i != 0) s += ", "; s += "\"" + json_values[i] + "\""; } s += "]}"; return s; } std::string RefSpec::ToString() const { return "RefSpec{uri=\"" + uri + "\"}"; } std::string AnyOfSpec::ToString() const { return "AnyOfSpec{options.size()=" + std::to_string(options.size()) + "}"; } std::string AllOfSpec::ToString() const { return "AllOfSpec{schemas.size()=" + std::to_string(schemas.size()) + "}"; } std::string TypeArraySpec::ToString() const { return "TypeArraySpec{type_schemas.size()=" + std::to_string(type_schemas.size()) + "}"; } std::string SchemaSpec::ToString() const { std::string spec_str; std::visit([&spec_str](const auto& s) { spec_str = s.ToString(); }, spec); return "SchemaSpec{spec=" + spec_str + ", cache_key=\"" + cache_key + "\", rule_name_hint=\"" + rule_name_hint + "\"}"; } // ==================== SchemaParser (Internal) ==================== namespace { enum class SchemaErrorType : int { kInvalidSchema = 0, kUnsatisfiableSchema = 1, }; using SchemaError = TypedError; /*! * \brief Parser for JSON Schema, converts JSON Schema to SchemaSpec intermediate representation. */ class SchemaParser { public: struct Config { bool strict_mode = false; JSONFormat json_format; }; explicit SchemaParser(const picojson::value& root_schema, const Config& config) : config_(config), root_schema_(root_schema) {} Result Parse( const picojson::value& schema, const std::string& rule_name_hint = "root", std::optional default_type = std::nullopt ); const picojson::value& GetRootSchema() const { return root_schema_; } bool IsStrictMode() const { return config_.strict_mode; } Result ResolveRef( const std::string& uri, const std::string& rule_name_hint ); private: Result ParseInteger(const picojson::object& schema); Result ParseNumber(const picojson::object& schema); Result ParseString(const picojson::object& schema); Result ParseBoolean(const picojson::object& schema); Result ParseNull(const picojson::object& schema); Result ParseArray(const picojson::object& schema); Result ParseObject(const picojson::object& schema); Result ParseConst(const picojson::object& schema); Result ParseEnum(const picojson::object& schema); Result ParseRef(const picojson::object& schema); Result ParseAnyOf(const picojson::object& schema); Result ParseAllOf(const picojson::object& schema); Result ParseTypeArray( const picojson::object& schema, const std::string& rule_name_hint ); std::string ComputeCacheKey(const picojson::value& schema); static void WarnUnsupportedKeywords( const picojson::object& schema, const std::vector& keywords, bool verbose = false ); Config config_; picojson::value root_schema_; std::unordered_map ref_cache_; std::unordered_map schema_cache_; }; std::string SchemaParser::ComputeCacheKey(const picojson::value& schema) { static const std::unordered_set kSkippedKeys = { "title", "default", "description", "examples", "deprecated", "readOnly", "writeOnly", "$comment", "$schema", }; if (schema.is()) { std::string result = "{"; std::vector> sorted_kv; for (const auto& kv : schema.get()) { if (kSkippedKeys.count(kv.first) == 0) { sorted_kv.push_back(kv); } } std::sort(sorted_kv.begin(), sorted_kv.end(), [](const auto& lhs, const auto& rhs) { return lhs.first < rhs.first; }); int64_t idx = 0; for (const auto& [key, value] : sorted_kv) { if (idx != 0) { result += ","; } ++idx; result += "\"" + key + "\":" + ComputeCacheKey(value); } return result + "}"; } else if (schema.is()) { std::string result = "["; int64_t idx = 0; for (const auto& item : schema.get()) { if (idx != 0) { result += ","; } ++idx; result += ComputeCacheKey(item); } return result + "]"; } return schema.serialize(false); } void SchemaParser::WarnUnsupportedKeywords( const picojson::object& schema, const std::vector& keywords, bool verbose ) { if (!verbose) { return; } for (const auto& keyword : keywords) { if (schema.find(keyword) != schema.end()) { XGRAMMAR_LOG(WARNING) << "Keyword " << keyword << " is not supported"; } } } Result SchemaParser::Parse( const picojson::value& schema, const std::string& rule_name_hint, std::optional default_type ) { std::string cache_key = ComputeCacheKey(schema); if (schema_cache_.count(cache_key)) { return ResultOk(schema_cache_[cache_key]); } if (schema.is()) { if (!schema.get()) { return ResultErr( SchemaErrorType::kUnsatisfiableSchema, "Schema 'false' cannot accept any value" ); } auto spec = SchemaSpec::Make(AnySpec{}, cache_key, rule_name_hint); schema_cache_[cache_key] = spec; return ResultOk(spec); } if (!schema.is()) { return ResultErr( SchemaErrorType::kInvalidSchema, "Schema should be an object or bool, but got " + schema.serialize(false) ); } const auto& schema_obj = schema.get(); WarnUnsupportedKeywords( schema_obj, {"not", "if", "then", "else", "dependentRequired", "dependentSchemas"} ); SchemaSpecPtr result; if (schema_obj.count("$ref")) { auto ref_result = ParseRef(schema_obj); if (ref_result.IsErr()) return ResultErr(std::move(ref_result).UnwrapErr()); auto ref_spec = std::move(ref_result).Unwrap(); result = SchemaSpec::Make(std::move(ref_spec), cache_key, rule_name_hint); } else if (schema_obj.count("const")) { auto const_result = ParseConst(schema_obj); if (const_result.IsErr()) return ResultErr(std::move(const_result).UnwrapErr()); result = SchemaSpec::Make(std::move(const_result).Unwrap(), cache_key, rule_name_hint); } else if (schema_obj.count("enum")) { auto enum_result = ParseEnum(schema_obj); if (enum_result.IsErr()) return ResultErr(std::move(enum_result).UnwrapErr()); result = SchemaSpec::Make(std::move(enum_result).Unwrap(), cache_key, rule_name_hint); } else if (schema_obj.count("anyOf") || schema_obj.count("oneOf")) { auto anyof_result = ParseAnyOf(schema_obj); if (anyof_result.IsErr()) return ResultErr(std::move(anyof_result).UnwrapErr()); result = SchemaSpec::Make(std::move(anyof_result).Unwrap(), cache_key, rule_name_hint); } else if (schema_obj.count("allOf")) { auto allof_result = ParseAllOf(schema_obj); if (allof_result.IsErr()) return ResultErr(std::move(allof_result).UnwrapErr()); result = SchemaSpec::Make(std::move(allof_result).Unwrap(), cache_key, rule_name_hint); } else if (schema_obj.count("type") || default_type.has_value()) { if (schema_obj.count("type") && schema_obj.at("type").is()) { auto type_array_result = ParseTypeArray(schema_obj, rule_name_hint); if (type_array_result.IsErr()) return ResultErr(std::move(type_array_result).UnwrapErr()); result = SchemaSpec::Make(std::move(type_array_result).Unwrap(), cache_key, rule_name_hint); } else { if (schema_obj.count("type") && !schema_obj.at("type").is()) { return ResultErr(SchemaErrorType::kInvalidSchema, "Type should be a string"); } const std::string& type = schema_obj.count("type") ? schema_obj.at("type").get() : default_type.value(); if (type == "integer") { auto int_result = ParseInteger(schema_obj); if (int_result.IsErr()) return ResultErr(std::move(int_result).UnwrapErr()); result = SchemaSpec::Make(std::move(int_result).Unwrap(), cache_key, rule_name_hint); } else if (type == "number") { auto num_result = ParseNumber(schema_obj); if (num_result.IsErr()) return ResultErr(std::move(num_result).UnwrapErr()); result = SchemaSpec::Make(std::move(num_result).Unwrap(), cache_key, rule_name_hint); } else if (type == "string") { auto str_result = ParseString(schema_obj); if (str_result.IsErr()) return ResultErr(std::move(str_result).UnwrapErr()); result = SchemaSpec::Make(std::move(str_result).Unwrap(), cache_key, rule_name_hint); } else if (type == "boolean") { auto bool_result = ParseBoolean(schema_obj); if (bool_result.IsErr()) return ResultErr(std::move(bool_result).UnwrapErr()); result = SchemaSpec::Make(std::move(bool_result).Unwrap(), cache_key, rule_name_hint); } else if (type == "null") { auto null_result = ParseNull(schema_obj); if (null_result.IsErr()) return ResultErr(std::move(null_result).UnwrapErr()); result = SchemaSpec::Make(std::move(null_result).Unwrap(), cache_key, rule_name_hint); } else if (type == "array") { auto array_result = ParseArray(schema_obj); if (array_result.IsErr()) return ResultErr(std::move(array_result).UnwrapErr()); result = SchemaSpec::Make(std::move(array_result).Unwrap(), cache_key, rule_name_hint); } else if (type == "object") { auto obj_result = ParseObject(schema_obj); if (obj_result.IsErr()) return ResultErr(std::move(obj_result).UnwrapErr()); result = SchemaSpec::Make(std::move(obj_result).Unwrap(), cache_key, rule_name_hint); } else { return ResultErr( SchemaErrorType::kInvalidSchema, "Unsupported type \"" + type + "\"" ); } } } else if (schema_obj.count("properties") || schema_obj.count("additionalProperties") || schema_obj.count("unevaluatedProperties")) { auto obj_result = ParseObject(schema_obj); if (obj_result.IsErr()) return ResultErr(std::move(obj_result).UnwrapErr()); result = SchemaSpec::Make(std::move(obj_result).Unwrap(), cache_key, rule_name_hint); } else if (schema_obj.count("items") || schema_obj.count("prefixItems") || schema_obj.count("unevaluatedItems")) { auto array_result = ParseArray(schema_obj); if (array_result.IsErr()) return ResultErr(std::move(array_result).UnwrapErr()); result = SchemaSpec::Make(std::move(array_result).Unwrap(), cache_key, rule_name_hint); } else { result = SchemaSpec::Make(AnySpec{}, cache_key, rule_name_hint); } schema_cache_[cache_key] = result; return ResultOk(result); } Result SchemaParser::ParseInteger(const picojson::object& schema) { WarnUnsupportedKeywords(schema, {"multipleOf"}); IntegerSpec spec; auto checkAndConvertIntegerBound = [](const picojson::value& value ) -> Result { if (!value.is() && !value.is()) { return ResultErr(SchemaErrorType::kInvalidSchema, "Value must be a number"); } if (value.is()) return ResultOk(value.get()); double val = value.get(); if (val != std::floor(val)) { return ResultErr( SchemaErrorType::kInvalidSchema, "Integer constraint must be a whole number" ); } static const double PROBLEMATIC_MIN = -9223372036854776000.0; static const double PROBLEMATIC_MAX = 9223372036854776000.0; if (val == PROBLEMATIC_MIN) { XGRAMMAR_CHECK(false ) << "Integer exceeds minimum limit due to precision loss at 64-bit boundary"; } if (val == PROBLEMATIC_MAX) { XGRAMMAR_CHECK(false ) << "Integer exceeds maximum limit due to precision loss at 64-bit boundary"; } static const double MAX_INT64_AS_DOUBLE = static_cast(std::numeric_limits::max()); static const double MIN_INT64_AS_DOUBLE = static_cast(std::numeric_limits::min()); XGRAMMAR_CHECK(val <= MAX_INT64_AS_DOUBLE) << "Integer exceeds maximum limit"; XGRAMMAR_CHECK(val >= MIN_INT64_AS_DOUBLE) << "Integer exceeds minimum limit"; return ResultOk(static_cast(val)); }; if (schema.count("minimum")) { auto result = checkAndConvertIntegerBound(schema.at("minimum")); if (result.IsErr()) return ResultErr(std::move(result).UnwrapErr()); spec.minimum = std::move(result).Unwrap(); } if (schema.count("maximum")) { auto result = checkAndConvertIntegerBound(schema.at("maximum")); if (result.IsErr()) return ResultErr(std::move(result).UnwrapErr()); spec.maximum = std::move(result).Unwrap(); } if (schema.count("exclusiveMinimum")) { auto result = checkAndConvertIntegerBound(schema.at("exclusiveMinimum")); if (result.IsErr()) return ResultErr(std::move(result).UnwrapErr()); int64_t val = std::move(result).Unwrap(); if (val == std::numeric_limits::max()) { return ResultErr( SchemaErrorType::kUnsatisfiableSchema, "exclusiveMinimum would cause integer overflow" ); } spec.exclusive_minimum = val; } if (schema.count("exclusiveMaximum")) { auto result = checkAndConvertIntegerBound(schema.at("exclusiveMaximum")); if (result.IsErr()) return ResultErr(std::move(result).UnwrapErr()); int64_t val = std::move(result).Unwrap(); if (val == std::numeric_limits::min()) { return ResultErr( SchemaErrorType::kUnsatisfiableSchema, "exclusiveMaximum would cause integer underflow" ); } spec.exclusive_maximum = val; } int64_t effective_min = spec.minimum.value_or(std::numeric_limits::min()); int64_t effective_max = spec.maximum.value_or(std::numeric_limits::max()); if (spec.exclusive_minimum.has_value()) { effective_min = std::max(effective_min, *spec.exclusive_minimum + 1); } if (spec.exclusive_maximum.has_value()) { effective_max = std::min(effective_max, *spec.exclusive_maximum - 1); } if (effective_min > effective_max) { return ResultErr( SchemaErrorType::kUnsatisfiableSchema, "Invalid range: minimum greater than maximum" ); } return ResultOk(std::move(spec)); } Result SchemaParser::ParseNumber(const picojson::object& schema) { WarnUnsupportedKeywords(schema, {"multipleOf"}); NumberSpec spec; auto getDouble = [](const picojson::value& value) -> Result { if (!value.is() && !value.is()) { return ResultErr(SchemaErrorType::kInvalidSchema, "Value must be a number"); } return ResultOk(value.get()); }; if (schema.count("minimum")) { auto result = getDouble(schema.at("minimum")); if (result.IsErr()) return ResultErr(std::move(result).UnwrapErr()); spec.minimum = std::move(result).Unwrap(); } if (schema.count("maximum")) { auto result = getDouble(schema.at("maximum")); if (result.IsErr()) return ResultErr(std::move(result).UnwrapErr()); spec.maximum = std::move(result).Unwrap(); } if (schema.count("exclusiveMinimum")) { auto result = getDouble(schema.at("exclusiveMinimum")); if (result.IsErr()) return ResultErr(std::move(result).UnwrapErr()); spec.exclusive_minimum = std::move(result).Unwrap(); } if (schema.count("exclusiveMaximum")) { auto result = getDouble(schema.at("exclusiveMaximum")); if (result.IsErr()) return ResultErr(std::move(result).UnwrapErr()); spec.exclusive_maximum = std::move(result).Unwrap(); } // The range is empty if any lower bound conflicts with any upper bound. An // exclusive bound also rules out equality, so it uses ">=" instead of ">". auto empty = []() { return ResultErr( SchemaErrorType::kUnsatisfiableSchema, "Invalid range: empty range" ); }; // minimum (x >= min) vs maximum (x <= max). if (spec.minimum && spec.maximum && *spec.minimum > *spec.maximum) { return empty(); } // minimum (x >= min) vs exclusiveMaximum (x < exclMax). if (spec.minimum && spec.exclusive_maximum && *spec.minimum >= *spec.exclusive_maximum) { return empty(); } // exclusiveMinimum (x > exclMin) vs maximum (x <= max). if (spec.exclusive_minimum && spec.maximum && *spec.exclusive_minimum >= *spec.maximum) { return empty(); } // exclusiveMinimum (x > exclMin) vs exclusiveMaximum (x < exclMax). if (spec.exclusive_minimum && spec.exclusive_maximum && *spec.exclusive_minimum >= *spec.exclusive_maximum) { return empty(); } return ResultOk(std::move(spec)); } Result SchemaParser::ParseString(const picojson::object& schema) { StringSpec spec; if (schema.count("format")) spec.format = schema.at("format").get(); if (schema.count("pattern")) spec.pattern = schema.at("pattern").get(); if (schema.count("minLength")) { if (!schema.at("minLength").is()) { return ResultErr( SchemaErrorType::kInvalidSchema, "minLength must be an integer" ); } spec.min_length = static_cast(schema.at("minLength").get()); } if (schema.count("maxLength")) { if (!schema.at("maxLength").is()) { return ResultErr( SchemaErrorType::kInvalidSchema, "maxLength must be an integer" ); } spec.max_length = static_cast(schema.at("maxLength").get()); } if (spec.max_length != -1 && spec.min_length > spec.max_length) { return ResultErr( SchemaErrorType::kUnsatisfiableSchema, "minLength " + std::to_string(spec.min_length) + " is greater than maxLength " + std::to_string(spec.max_length) ); } return ResultOk(std::move(spec)); } Result SchemaParser::ParseBoolean(const picojson::object&) { return ResultOk(BooleanSpec{}); } Result SchemaParser::ParseNull(const picojson::object&) { return ResultOk(NullSpec{}); } Result SchemaParser::ParseArray(const picojson::object& schema) { WarnUnsupportedKeywords(schema, {"uniqueItems", "contains", "minContains", "maxContains"}); ArraySpec spec; if (schema.count("prefixItems")) { if (!schema.at("prefixItems").is()) { return ResultErr( SchemaErrorType::kInvalidSchema, "prefixItems must be an array" ); } for (const auto& item : schema.at("prefixItems").get()) { if (item.is() && !item.get()) { return ResultErr( SchemaErrorType::kUnsatisfiableSchema, "prefixItems contains false" ); } else if (!item.is()) { return ResultErr( SchemaErrorType::kInvalidSchema, "prefixItems must be an array of objects or booleans" ); } auto item_result = Parse(item, "prefix_item"); if (item_result.IsErr()) return ResultErr(std::move(item_result).UnwrapErr()); spec.prefix_items.push_back(std::move(item_result).Unwrap()); } } if (schema.count("items")) { auto items_value = schema.at("items"); if (!items_value.is() && !items_value.is()) { return ResultErr( SchemaErrorType::kInvalidSchema, "items must be a boolean or an object" ); } if (items_value.is() && !items_value.get()) { spec.allow_additional_items = false; } else { spec.allow_additional_items = true; auto items_result = Parse(items_value, "item"); if (items_result.IsErr()) return ResultErr(std::move(items_result).UnwrapErr()); spec.additional_items = std::move(items_result).Unwrap(); } } else if (schema.count("unevaluatedItems")) { auto unevaluated_items_value = schema.at("unevaluatedItems"); if (!unevaluated_items_value.is() && !unevaluated_items_value.is()) { return ResultErr( SchemaErrorType::kInvalidSchema, "unevaluatedItems must be a boolean or an object" ); } if (unevaluated_items_value.is() && !unevaluated_items_value.get()) { spec.allow_additional_items = false; } else { spec.allow_additional_items = true; auto items_result = Parse(unevaluated_items_value, "unevaluated_item"); if (items_result.IsErr()) return ResultErr(std::move(items_result).UnwrapErr()); spec.additional_items = std::move(items_result).Unwrap(); } } else if (!config_.strict_mode) { spec.allow_additional_items = true; spec.additional_items = SchemaSpec::Make(AnySpec{}, "", "any"); } else { spec.allow_additional_items = false; } if (schema.count("minItems")) { if (!schema.at("minItems").is()) { return ResultErr(SchemaErrorType::kInvalidSchema, "minItems must be an integer"); } spec.min_items = std::max(static_cast(0), schema.at("minItems").get()); } if (schema.count("minContains")) { if (!schema.at("minContains").is()) { return ResultErr( SchemaErrorType::kInvalidSchema, "minContains must be an integer" ); } spec.min_items = std::max(spec.min_items, schema.at("minContains").get()); } if (schema.count("maxItems")) { if (!schema.at("maxItems").is() || schema.at("maxItems").get() < 0) { return ResultErr( SchemaErrorType::kInvalidSchema, "maxItems must be a non-negative integer" ); } spec.max_items = schema.at("maxItems").get(); } if (spec.max_items != -1 && spec.min_items > spec.max_items) { return ResultErr( SchemaErrorType::kUnsatisfiableSchema, "minItems is greater than maxItems: " + std::to_string(spec.min_items) + " > " + std::to_string(spec.max_items) ); } if (spec.max_items != -1 && spec.max_items < static_cast(spec.prefix_items.size())) { return ResultErr( SchemaErrorType::kUnsatisfiableSchema, "maxItems is less than the number of prefixItems: " + std::to_string(spec.max_items) + " < " + std::to_string(spec.prefix_items.size()) ); } if (!spec.allow_additional_items) { int64_t prefix_size = static_cast(spec.prefix_items.size()); if (prefix_size < spec.min_items) { return ResultErr( SchemaErrorType::kUnsatisfiableSchema, "minItems is greater than the number of prefixItems, but additional items are not " "allowed: " + std::to_string(spec.min_items) + " > " + std::to_string(prefix_size) ); } if (spec.max_items != -1 && prefix_size > spec.max_items) { return ResultErr( SchemaErrorType::kUnsatisfiableSchema, "maxItems is less than the number of prefixItems, but additional items are not " "allowed: " + std::to_string(spec.max_items) + " < " + std::to_string(prefix_size) ); } } return ResultOk(std::move(spec)); } Result SchemaParser::ParseObject(const picojson::object& schema) { ObjectSpec spec; if (schema.count("properties")) { if (!schema.at("properties").is()) { return ResultErr( SchemaErrorType::kInvalidSchema, "properties must be an object" ); } auto properties_obj = schema.at("properties").get(); for (const auto& key : properties_obj.ordered_keys()) { auto prop_result = Parse(properties_obj.at(key), key); if (prop_result.IsErr()) return ResultErr(std::move(prop_result).UnwrapErr()); spec.properties.push_back({key, std::move(prop_result).Unwrap()}); } } if (schema.count("required")) { if (!schema.at("required").is()) { return ResultErr(SchemaErrorType::kInvalidSchema, "required must be an array"); } for (const auto& req : schema.at("required").get()) { spec.required.insert(req.get()); } } if (schema.count("patternProperties")) { if (!schema.at("patternProperties").is()) { return ResultErr( SchemaErrorType::kInvalidSchema, "patternProperties must be an object" ); } auto pattern_props = schema.at("patternProperties").get(); for (const auto& key : pattern_props.ordered_keys()) { auto prop_result = Parse(pattern_props.at(key), "pattern_prop"); if (prop_result.IsErr()) return ResultErr(std::move(prop_result).UnwrapErr()); spec.pattern_properties.push_back({key, std::move(prop_result).Unwrap()}); } } if (schema.count("propertyNames")) { if (!schema.at("propertyNames").is()) { return ResultErr( SchemaErrorType::kInvalidSchema, "propertyNames must be an object" ); } auto property_names_obj = schema.at("propertyNames").get(); if (property_names_obj.count("type") && property_names_obj.at("type").is() && property_names_obj.at("type").get() != "string") { return ResultErr( SchemaErrorType::kUnsatisfiableSchema, "propertyNames must be an object that validates string" ); } auto prop_names_result = Parse(schema.at("propertyNames"), "property_name", "string"); if (prop_names_result.IsErr()) return ResultErr(std::move(prop_names_result).UnwrapErr()); spec.property_names = std::move(prop_names_result).Unwrap(); } spec.allow_additional_properties = !config_.strict_mode; if (schema.count("additionalProperties")) { auto add_props = schema.at("additionalProperties"); if (add_props.is()) { spec.allow_additional_properties = add_props.get(); } else { spec.allow_additional_properties = true; auto add_props_result = Parse(add_props, "additional"); if (add_props_result.IsErr()) return ResultErr(std::move(add_props_result).UnwrapErr()); spec.additional_properties_schema = std::move(add_props_result).Unwrap(); } } spec.allow_unevaluated_properties = true; if (schema.count("additionalProperties")) { spec.allow_unevaluated_properties = spec.allow_additional_properties; } else if (schema.count("unevaluatedProperties")) { auto uneval_props = schema.at("unevaluatedProperties"); if (uneval_props.is()) { spec.allow_unevaluated_properties = uneval_props.get(); } else { spec.allow_unevaluated_properties = true; auto uneval_result = Parse(uneval_props, "unevaluated"); if (uneval_result.IsErr()) return ResultErr(std::move(uneval_result).UnwrapErr()); spec.unevaluated_properties_schema = std::move(uneval_result).Unwrap(); } } else if (config_.strict_mode) { spec.allow_unevaluated_properties = false; } if (schema.count("minProperties")) { if (!schema.at("minProperties").is()) { return ResultErr( SchemaErrorType::kInvalidSchema, "minProperties must be an integer" ); } spec.min_properties = static_cast(schema.at("minProperties").get()); if (spec.min_properties < 0) { return ResultErr( SchemaErrorType::kUnsatisfiableSchema, "minProperties must be a non-negative integer" ); } } if (schema.count("maxProperties")) { if (!schema.at("maxProperties").is()) { return ResultErr( SchemaErrorType::kInvalidSchema, "maxProperties must be an integer" ); } spec.max_properties = static_cast(schema.at("maxProperties").get()); if (spec.max_properties < 0) { return ResultErr( SchemaErrorType::kUnsatisfiableSchema, "maxProperties must be a non-negative integer" ); } } if (spec.max_properties != -1 && spec.min_properties > spec.max_properties) { return ResultErr( SchemaErrorType::kUnsatisfiableSchema, "minProperties is greater than maxProperties: " + std::to_string(spec.min_properties) + " > " + std::to_string(spec.max_properties) ); } if (spec.max_properties != -1 && static_cast(spec.required.size()) > spec.max_properties) { return ResultErr( SchemaErrorType::kUnsatisfiableSchema, "maxProperties is less than the number of required properties: " + std::to_string(spec.max_properties) + " < " + std::to_string(spec.required.size()) ); } if (spec.pattern_properties.empty() && !spec.property_names && !spec.allow_additional_properties && !spec.allow_unevaluated_properties && spec.min_properties > static_cast(spec.properties.size())) { return ResultErr( SchemaErrorType::kUnsatisfiableSchema, "minProperties is greater than the number of properties, but additional properties aren't " "allowed: " + std::to_string(spec.min_properties) + " > " + std::to_string(spec.properties.size()) ); } return ResultOk(std::move(spec)); } Result SchemaParser::ParseConst(const picojson::object& schema) { ConstSpec spec; spec.json_value = schema.at("const").serialize(); return ResultOk(std::move(spec)); } Result SchemaParser::ParseEnum(const picojson::object& schema) { EnumSpec spec; if (!schema.at("enum").is()) { return ResultErr(SchemaErrorType::kInvalidSchema, "enum must be an array"); } const auto& enum_array = schema.at("enum").get(); if (enum_array.empty()) { return ResultErr(SchemaErrorType::kInvalidSchema, "enum array must not be empty"); } for (const auto& value : enum_array) { spec.json_values.push_back(value.serialize()); } return ResultOk(std::move(spec)); } Result SchemaParser::ParseRef(const picojson::object& schema) { if (!schema.at("$ref").is()) { return ResultErr(SchemaErrorType::kInvalidSchema, "$ref must be a string"); } RefSpec spec; spec.uri = schema.at("$ref").get(); return ResultOk(std::move(spec)); } Result SchemaParser::ResolveRef( const std::string& uri, const std::string& rule_name_hint ) { if (ref_cache_.count(uri)) return ResultOk(ref_cache_[uri]); if (uri == "#") { auto placeholder = SchemaSpec::Make(AnySpec{}, "", "root"); ref_cache_[uri] = placeholder; auto result = Parse(root_schema_, "root"); if (result.IsErr()) return ResultErr(std::move(result).UnwrapErr()); auto resolved = std::move(result).Unwrap(); ref_cache_[uri] = resolved; return ResultOk(resolved); } if (uri.size() < 2 || uri[0] != '#' || uri[1] != '/') { XGRAMMAR_LOG(WARNING) << "URI should either be '#' or start with '#/' but got " << uri; return ResultOk(SchemaSpec::Make(AnySpec{}, "", "any")); } std::vector parts; std::stringstream ss(uri.substr(2)); std::string part; std::string new_rule_name_prefix; while (std::getline(ss, part, '/')) { if (!part.empty()) parts.push_back(part); if (!new_rule_name_prefix.empty()) new_rule_name_prefix += "_"; for (const auto& c : part) { if (std::isalpha(c) || c == '_' || c == '-' || c == '.') new_rule_name_prefix += c; } } auto current = std::cref(root_schema_); for (const auto& p : parts) { if (!current.get().is() || !current.get().contains(p)) { return ResultErr( SchemaErrorType::kInvalidSchema, "Cannot find field " + p + " in " + uri ); } current = current.get().get(p); } auto result = Parse(current, new_rule_name_prefix); if (result.IsErr()) return ResultErr(std::move(result).UnwrapErr()); auto resolved = std::move(result).Unwrap(); ref_cache_[uri] = resolved; return ResultOk(resolved); } Result SchemaParser::ParseAnyOf(const picojson::object& schema) { AnyOfSpec spec; auto anyof_key = schema.count("anyOf") ? "anyOf" : "oneOf"; if (!schema.at(anyof_key).is()) { return ResultErr( SchemaErrorType::kInvalidSchema, std::string(anyof_key) + " must be an array" ); } int idx = 0; for (const auto& option : schema.at(anyof_key).get()) { auto option_result = Parse(option, "case_" + std::to_string(idx)); if (option_result.IsErr()) return ResultErr(std::move(option_result).UnwrapErr()); spec.options.push_back(std::move(option_result).Unwrap()); ++idx; } return ResultOk(std::move(spec)); } Result SchemaParser::ParseAllOf(const picojson::object& schema) { AllOfSpec spec; if (!schema.at("allOf").is()) { return ResultErr(SchemaErrorType::kInvalidSchema, "allOf must be an array"); } int idx = 0; for (const auto& sub_schema : schema.at("allOf").get()) { auto sub_result = Parse(sub_schema, "all_" + std::to_string(idx)); if (sub_result.IsErr()) return ResultErr(std::move(sub_result).UnwrapErr()); spec.schemas.push_back(std::move(sub_result).Unwrap()); ++idx; } return ResultOk(std::move(spec)); } Result SchemaParser::ParseTypeArray( const picojson::object& schema, const std::string& rule_name_hint ) { TypeArraySpec spec; auto type_array = schema.at("type").get(); picojson::object schema_copy = schema; if (type_array.empty()) { schema_copy.erase("type"); auto any_result = Parse(picojson::value(schema_copy), rule_name_hint); if (any_result.IsErr()) return ResultErr(std::move(any_result).UnwrapErr()); spec.type_schemas.push_back(std::move(any_result).Unwrap()); return ResultOk(std::move(spec)); } for (const auto& type : type_array) { if (!type.is()) { return ResultErr( SchemaErrorType::kInvalidSchema, "type must be a string or an array of strings" ); } schema_copy["type"] = type; auto type_result = Parse(picojson::value(schema_copy), rule_name_hint + "_" + type.get()); if (type_result.IsErr()) return ResultErr(std::move(type_result).UnwrapErr()); spec.type_schemas.push_back(std::move(type_result).Unwrap()); } return ResultOk(std::move(spec)); } } // namespace // ==================== IndentManager Implementation ==================== IndentManager::IndentManager( std::optional indent, const std::string& separator, bool any_whitespace, std::optional max_whitespace_cnt ) : any_whitespace_(any_whitespace), enable_newline_(indent.has_value()), indent_(indent.value_or(0)), separator_(separator), total_indent_(0), is_first_({true}), max_whitespace_cnt_(max_whitespace_cnt) { if (max_whitespace_cnt.has_value() && max_whitespace_cnt.value() <= 0) { XGRAMMAR_LOG(FATAL) << "max_whitespace_cnt must be positive."; } } void IndentManager::StartIndent() { total_indent_ += indent_; is_first_.push_back(true); } void IndentManager::EndIndent() { total_indent_ -= indent_; is_first_.pop_back(); } std::string IndentManager::StartSeparator() { if (any_whitespace_) { if (!max_whitespace_cnt_.has_value()) { return "[ \\n\\t]*"; } else { return "[ \\n\\t]{0," + std::to_string(max_whitespace_cnt_.value()) + "}"; } } if (!enable_newline_) { return "\"\""; } return "\"\\n" + std::string(total_indent_, ' ') + "\""; } std::string IndentManager::MiddleSeparator() { if (any_whitespace_) { std::string whitespace_part; if (!max_whitespace_cnt_.has_value()) { whitespace_part = "[ \\n\\t]*"; } else { whitespace_part = "[ \\n\\t]{0," + std::to_string(max_whitespace_cnt_.value()) + "}"; } return whitespace_part + " \"" + separator_ + "\" " + whitespace_part; } if (!enable_newline_) { return "\"" + separator_ + "\""; } return "\"" + separator_ + "\\n" + std::string(total_indent_, ' ') + "\""; } std::string IndentManager::EndSeparator() { if (any_whitespace_) { if (!max_whitespace_cnt_.has_value()) { return "[ \\n\\t]*"; } else { return "[ \\n\\t]{0," + std::to_string(max_whitespace_cnt_.value()) + "}"; } } if (!enable_newline_) { return "\"\""; } return "\"\\n" + std::string(total_indent_ - indent_, ' ') + "\""; } std::string IndentManager::EmptySeparator() { if (any_whitespace_) { if (!max_whitespace_cnt_.has_value()) { return "[ \\n\\t]*"; } else { return "[ \\n\\t]{0," + std::to_string(max_whitespace_cnt_.value()) + "}"; } } return "\"\""; } std::string IndentManager::NextSeparator(bool is_end) { if (any_whitespace_) { if (is_first_.back() || is_end) { is_first_.back() = false; if (!max_whitespace_cnt_.has_value()) { return "[ \\n\\t]*"; } else { return "[ \\n\\t]{0," + std::to_string(max_whitespace_cnt_.value()) + "}"; } } else { std::string whitespace_part; if (!max_whitespace_cnt_.has_value()) { whitespace_part = "[ \\n\\t]*"; } else { whitespace_part = "[ \\n\\t]{0," + std::to_string(max_whitespace_cnt_.value()) + "}"; } return whitespace_part + " \"" + separator_ + "\" " + whitespace_part; } } std::string res = ""; if (!is_first_.back() && !is_end) { res += separator_; } is_first_.back() = false; if (enable_newline_) { res += "\\n"; } if (!is_end) { res += std::string(total_indent_, ' '); } else { res += std::string(total_indent_ - indent_, ' '); } return "\"" + res + "\""; } // ==================== Static Constants ==================== const std::string JSONSchemaConverter::kBasicAny = "basic_any"; const std::string JSONSchemaConverter::kBasicInteger = "basic_integer"; const std::string JSONSchemaConverter::kBasicNumber = "basic_number"; const std::string JSONSchemaConverter::kBasicString = "basic_string"; const std::string JSONSchemaConverter::kBasicBoolean = "basic_boolean"; const std::string JSONSchemaConverter::kBasicNull = "basic_null"; const std::string JSONSchemaConverter::kBasicArray = "basic_array"; const std::string JSONSchemaConverter::kBasicObject = "basic_object"; const std::string JSONSchemaConverter::kBasicEscape = "basic_escape"; const std::string JSONSchemaConverter::kBasicStringSub = "basic_string_sub"; // ==================== JSONSchemaConverter Implementation ==================== JSONSchemaConverter::JSONSchemaConverter( std::optional indent, std::optional> separators, bool any_whitespace, std::optional max_whitespace_cnt, RefResolver ref_resolver, bool any_order ) : indent_manager_( indent, separators.has_value() ? separators->first : (any_whitespace ? "," : (indent.has_value() ? "," : ", ")), any_whitespace, max_whitespace_cnt ), any_whitespace_(any_whitespace), max_whitespace_cnt_(max_whitespace_cnt), any_order_(any_order), ref_resolver_(std::move(ref_resolver)) { std::string colon_sep = separators.has_value() ? separators->second : (any_whitespace ? ":" : ": "); if (any_whitespace) { std::string whitespace_part; if (!max_whitespace_cnt_.has_value()) { whitespace_part = "[ \\n\\t]*"; } else { whitespace_part = "[ \\n\\t]{0," + std::to_string(max_whitespace_cnt_.value()) + "}"; } colon_pattern_ = whitespace_part + " \"" + colon_sep + "\" " + whitespace_part; } else { colon_pattern_ = "\"" + colon_sep + "\""; } } std::string JSONSchemaConverter::Convert(const SchemaSpecPtr& spec) { AddBasicRules(); // Register the root rule for circular reference handling // This allows $ref: "#" to resolve to "root" std::string root_rule_name = ebnf_script_creator_.AllocateRuleName("root"); uri_to_rule_name_["#"] = root_rule_name; // Check if the spec can be directly mapped to an existing rule auto cached_rule = GetCache(spec->cache_key); if (cached_rule.has_value()) { // Root schema matches a basic type, just reference it ebnf_script_creator_.AddRuleWithAllocatedName(root_rule_name, cached_rule.value()); } else { // Generate the rule body if (!spec->cache_key.empty()) { AddCache(spec->cache_key, root_rule_name); } std::string root_body = GenerateFromSpec(spec, root_rule_name); ebnf_script_creator_.AddRuleWithAllocatedName(root_rule_name, root_body); } return ebnf_script_creator_.GetScript(); } void JSONSchemaConverter::AddBasicRules() { AddHelperRules(); // Create basic rules with a temporary indent manager for compact format auto saved_indent_manager = indent_manager_; if (any_whitespace_) { indent_manager_ = IndentManager(std::nullopt, ",", true, std::nullopt); } else { indent_manager_ = IndentManager(std::nullopt, ", ", false, std::nullopt); } // basic_any - use "{}" as the cache key for empty schema auto any_spec = SchemaSpec::Make(AnySpec{}, "{}", kBasicAny); std::string any_body = GenerateAny(std::get(any_spec->spec), kBasicAny); ebnf_script_creator_.AddRule(kBasicAny, any_body); AddCache("{}", kBasicAny); // basic_integer - cache_key matches SchemaParser::ComputeCacheKey for {"type": "integer"} constexpr const char* kIntegerCacheKey = "{\"type\":\"integer\"}"; auto int_spec = SchemaSpec::Make(IntegerSpec{}, kIntegerCacheKey, kBasicInteger); std::string int_body = GenerateInteger(std::get(int_spec->spec), kBasicInteger); ebnf_script_creator_.AddRule(kBasicInteger, int_body); AddCache(kIntegerCacheKey, kBasicInteger); // basic_number - cache_key matches SchemaParser::ComputeCacheKey for {"type": "number"} constexpr const char* kNumberCacheKey = "{\"type\":\"number\"}"; auto num_spec = SchemaSpec::Make(NumberSpec{}, kNumberCacheKey, kBasicNumber); std::string num_body = GenerateNumber(std::get(num_spec->spec), kBasicNumber); ebnf_script_creator_.AddRule(kBasicNumber, num_body); AddCache(kNumberCacheKey, kBasicNumber); // basic_string - cache_key matches SchemaParser::ComputeCacheKey for {"type": "string"} constexpr const char* kStringCacheKey = "{\"type\":\"string\"}"; auto str_spec = SchemaSpec::Make(StringSpec{}, kStringCacheKey, kBasicString); std::string str_body = "[\"] " + kBasicStringSub; ebnf_script_creator_.AddRule(kBasicString, str_body); AddCache(kStringCacheKey, kBasicString); // basic_boolean - cache_key matches SchemaParser::ComputeCacheKey for {"type": "boolean"} constexpr const char* kBooleanCacheKey = "{\"type\":\"boolean\"}"; auto bool_spec = SchemaSpec::Make(BooleanSpec{}, kBooleanCacheKey, kBasicBoolean); std::string bool_body = GenerateBoolean(std::get(bool_spec->spec), kBasicBoolean); ebnf_script_creator_.AddRule(kBasicBoolean, bool_body); AddCache(kBooleanCacheKey, kBasicBoolean); // basic_null - cache_key matches SchemaParser::ComputeCacheKey for {"type": "null"} constexpr const char* kNullCacheKey = "{\"type\":\"null\"}"; auto null_spec = SchemaSpec::Make(NullSpec{}, kNullCacheKey, kBasicNull); std::string null_body = GenerateNull(std::get(null_spec->spec), kBasicNull); ebnf_script_creator_.AddRule(kBasicNull, null_body); AddCache(kNullCacheKey, kBasicNull); // basic_array - cache_key matches SchemaParser::ComputeCacheKey for {"type": "array"} constexpr const char* kArrayCacheKey = "{\"type\":\"array\"}"; ArraySpec array_spec_val; array_spec_val.allow_additional_items = true; array_spec_val.additional_items = any_spec; auto array_spec = SchemaSpec::Make(std::move(array_spec_val), kArrayCacheKey, kBasicArray); std::string array_body = GenerateArray(std::get(array_spec->spec), kBasicArray); ebnf_script_creator_.AddRule(kBasicArray, array_body); AddCache(kArrayCacheKey, kBasicArray); // basic_object - cache_key matches SchemaParser::ComputeCacheKey for {"type": "object"} constexpr const char* kObjectCacheKey = "{\"type\":\"object\"}"; ObjectSpec obj_spec_val; obj_spec_val.allow_additional_properties = true; obj_spec_val.additional_properties_schema = any_spec; auto obj_spec = SchemaSpec::Make(std::move(obj_spec_val), kObjectCacheKey, kBasicObject); std::string obj_body = GenerateObject(std::get(obj_spec->spec), kBasicObject); ebnf_script_creator_.AddRule(kBasicObject, obj_body); AddCache(kObjectCacheKey, kBasicObject); indent_manager_ = saved_indent_manager; } void JSONSchemaConverter::AddHelperRules() { ebnf_script_creator_.AddRule( kBasicEscape, "[\"\\\\/bfnrt] | \"u\" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9]" ); std::string whitespace_part = GetWhitespacePattern(); ebnf_script_creator_.AddRule( kBasicStringSub, "(\"\\\"\" | [^\\0-\\x1f\\\"\\\\\\r\\n] " + kBasicStringSub + " | \"\\\\\" " + kBasicEscape + " " + kBasicStringSub + ") (= " + whitespace_part + " [,}\\]:])" ); } std::string JSONSchemaConverter::GetWhitespacePattern() const { if (!max_whitespace_cnt_.has_value()) { return "[ \\n\\t]*"; } else { return "[ \\n\\t]{0," + std::to_string(max_whitespace_cnt_.value()) + "}"; } } std::string JSONSchemaConverter::NextSeparator(bool is_end) { return indent_manager_.NextSeparator(is_end); } std::string JSONSchemaConverter::GetKeyPattern() const { return kBasicString; } namespace { struct TrieNode { bool is_terminal = false; std::map children; }; std::string BuildTrieBody(const TrieNode& node) { std::string result; bool first = true; auto add = [&](const std::string& s) { if (!first) result += " | "; first = false; result += s; }; // 1. Close quote - only if no excluded key ends here if (!node.is_terminal) { add("\"\\\"\""); } // 2. Negated char class - excludes edge chars + JSON specials std::string neg = "[^"; for (const auto& [c, _] : node.children) { if (c == ']' || c == '\\' || c == '^' || c == '-') { neg += "\\"; } neg += c; } neg += "\\0-\\x1f\\\"\\\\\\r\\n]"; add(neg + " " + JSONSchemaConverter::kBasicStringSub); // 3. Escape sequence add("\"\\\\\" " + JSONSchemaConverter::kBasicEscape + " " + JSONSchemaConverter::kBasicStringSub); // 4. Trie edges - recurse for (const auto& [c, child] : node.children) { std::string child_body = BuildTrieBody(child); std::string char_lit = "\""; if (c == '"') { char_lit += "\\\""; } else if (c == '\\') { char_lit += "\\\\"; } else { char_lit += c; } char_lit += "\""; add(char_lit + " " + child_body); } return "(" + result + ")"; } } // namespace std::string JSONSchemaConverter::GetKeyPatternExcluding( const std::vector& properties, const std::string& rule_name ) { if (properties.empty()) { return GetKeyPattern(); } // Build trie from property names // TODO(linzhang): The trie only excludes the literal unescaped spelling of each property name. TrieNode root; for (const auto& prop : properties) { TrieNode* cur = &root; for (char c : prop.name) { cur = &cur->children[c]; } cur->is_terminal = true; } // Generate EBNF body std::string inner = BuildTrieBody(root); std::string ws = GetWhitespacePattern(); std::string body = "[\"] (" + inner + ") (= " + ws + " [,}\\]:])"; return ebnf_script_creator_.AddRule(rule_name + "_addl_key", body); } std::string JSONSchemaConverter::GetBasicAnyRuleName() const { return kBasicAny; } void JSONSchemaConverter::AddCache(const std::string& key, const std::string& value) { if (key.empty()) { return; } rule_cache_manager_.AddCache(key, true, value); } std::optional JSONSchemaConverter::GetCache(const std::string& key) const { if (key.empty()) { return std::nullopt; } return rule_cache_manager_.GetCache(key, true); } std::string JSONSchemaConverter::CreateRule( const SchemaSpecPtr& spec, const std::string& rule_name_hint ) { // Only check cache for basic rules (pre-populated in AddBasicRules) // Don't cache other rules to match original behavior auto cached = GetCache(spec->cache_key); if (cached.has_value()) { return cached.value(); } std::string rule_name = ebnf_script_creator_.AllocateRuleName(rule_name_hint); std::string rule_body = GenerateFromSpec(spec, rule_name); ebnf_script_creator_.AddRuleWithAllocatedName(rule_name, rule_body); return rule_name; } std::string JSONSchemaConverter::GenerateFromSpec( const SchemaSpecPtr& spec, const std::string& rule_name_hint ) { return std::visit( [this, &rule_name_hint](const auto& s) -> std::string { using T = std::decay_t; if constexpr (std::is_same_v) { return GenerateInteger(s, rule_name_hint); } else if constexpr (std::is_same_v) { return GenerateNumber(s, rule_name_hint); } else if constexpr (std::is_same_v) { return GenerateString(s, rule_name_hint); } else if constexpr (std::is_same_v) { return GenerateBoolean(s, rule_name_hint); } else if constexpr (std::is_same_v) { return GenerateNull(s, rule_name_hint); } else if constexpr (std::is_same_v) { return GenerateArray(s, rule_name_hint); } else if constexpr (std::is_same_v) { return GenerateObject(s, rule_name_hint); } else if constexpr (std::is_same_v) { return GenerateAny(s, rule_name_hint); } else if constexpr (std::is_same_v) { return GenerateConst(s, rule_name_hint); } else if constexpr (std::is_same_v) { return GenerateEnum(s, rule_name_hint); } else if constexpr (std::is_same_v) { return GenerateRef(s, rule_name_hint); } else if constexpr (std::is_same_v) { return GenerateAnyOf(s, rule_name_hint); } else if constexpr (std::is_same_v) { return GenerateAllOf(s, rule_name_hint); } else if constexpr (std::is_same_v) { return GenerateTypeArray(s, rule_name_hint); } else { XGRAMMAR_LOG(FATAL) << "Unknown spec type"; return ""; } }, spec->spec ); } // ==================== Generate Methods ==================== std::string JSONSchemaConverter::GenerateInteger( const IntegerSpec& spec, const std::string& rule_name ) { std::optional start, end; if (spec.minimum.has_value()) { start = spec.minimum; } if (spec.exclusive_minimum.has_value()) { // Smallest integer strictly greater than exclusive_minimum (the parser // rejects exclusive_minimum == INT64_MAX, so +1 cannot overflow). When // minimum is also present the stricter (larger) lower bound wins. int64_t excl_start = *spec.exclusive_minimum + 1; start = start.has_value() ? std::max(*start, excl_start) : excl_start; } if (spec.maximum.has_value()) { end = spec.maximum; } if (spec.exclusive_maximum.has_value()) { // Largest integer strictly less than exclusive_maximum (the parser rejects // exclusive_maximum == INT64_MIN, so -1 cannot underflow). When maximum is // also present the stricter (smaller) upper bound wins. int64_t excl_end = *spec.exclusive_maximum - 1; end = end.has_value() ? std::min(*end, excl_end) : excl_end; } if (start.has_value() || end.has_value()) { std::string range_regex = GenerateRangeRegex(start, end); return RegexToEBNF(range_regex, false); } return "(\"0\" | \"-\"? [1-9] [0-9]*)"; } std::string JSONSchemaConverter::GenerateNumber( const NumberSpec& spec, const std::string& rule_name ) { std::optional start, end; bool exclusive_start = false; bool exclusive_end = false; if (spec.minimum.has_value()) { start = spec.minimum; } // When both bounds are present the larger lower bound wins; on a tie the // exclusive one is stricter. if (spec.exclusive_minimum.has_value() && (!start.has_value() || *spec.exclusive_minimum >= *start)) { start = spec.exclusive_minimum; exclusive_start = true; } if (spec.maximum.has_value()) { end = spec.maximum; } if (spec.exclusive_maximum.has_value() && (!end.has_value() || *spec.exclusive_maximum <= *end)) { end = spec.exclusive_maximum; exclusive_end = true; } if (start.has_value() || end.has_value()) { std::string range_regex = GenerateFloatRangeRegex(start, end, 6, exclusive_start, exclusive_end); return RegexToEBNF(range_regex, false); } // Note: The format must be "-"? ("0" | ...) not ("0" | "-"? ...) // The first allows -0, -123, 0, 123 // The second allows 0, -123, 123 but not -0 return "\"-\"? (\"0\" | [1-9] [0-9]*) (\".\" [0-9]+)? ([eE] [+-]? [0-9]+)?"; } std::string JSONSchemaConverter::GenerateString( const StringSpec& spec, const std::string& rule_name ) { // Check for format if (spec.format.has_value()) { const std::string& format = *spec.format; auto regex_pattern = JSONFormatToRegexPattern(format); if (regex_pattern.has_value()) { std::string converted_regex = RegexToEBNF(regex_pattern.value(), false); return "\"\\\"\" " + converted_regex + " \"\\\"\""; } } // Check for pattern if (spec.pattern.has_value()) { std::string converted_regex = RegexToEBNF(*spec.pattern, false); return "\"\\\"\" " + converted_regex + " \"\\\"\""; } // Check for length constraints if (spec.min_length != 0 || spec.max_length != -1) { std::string char_pattern = "[^\"\\\\\\r\\n]"; std::string repetition; if (spec.max_length == -1) { repetition = "{" + std::to_string(spec.min_length) + ",}"; } else { repetition = "{" + std::to_string(spec.min_length) + "," + std::to_string(spec.max_length) + "}"; } return "\"\\\"\" " + char_pattern + repetition + " \"\\\"\""; } // Default string return "[\"] " + kBasicStringSub; } std::string JSONSchemaConverter::GenerateBoolean( const BooleanSpec& spec, const std::string& rule_name ) { return "\"true\" | \"false\""; } std::string JSONSchemaConverter::GenerateNull(const NullSpec& spec, const std::string& rule_name) { return "\"null\""; } std::string JSONSchemaConverter::GenerateArray( const ArraySpec& spec, const std::string& rule_name ) { indent_manager_.StartIndent(); auto start_separator = indent_manager_.StartSeparator(); auto mid_separator = indent_manager_.MiddleSeparator(); auto end_separator = indent_manager_.EndSeparator(); auto empty_separator = indent_manager_.EmptySeparator(); std::vector item_rule_names; std::string additional_rule_name; // Handle prefix items for (size_t i = 0; i < spec.prefix_items.size(); ++i) { item_rule_names.push_back( CreateRule(spec.prefix_items[i], rule_name + "_item_" + std::to_string(i)) ); } // Handle additional items if (spec.allow_additional_items && spec.additional_items) { additional_rule_name = CreateRule(spec.additional_items, rule_name + "_additional"); } indent_manager_.EndIndent(); // Construct the result const std::string& left_bracket = EBNFScriptCreator::Str("["); const std::string& right_bracket = EBNFScriptCreator::Str("]"); if (spec.prefix_items.empty()) { auto empty_part = EBNFScriptCreator::Concat({left_bracket, empty_separator, right_bracket}); if (!spec.allow_additional_items) { return empty_part; } else if (spec.min_items == 0 && spec.max_items == 0) { return empty_part; } else if (spec.min_items == 0 && spec.max_items != 0) { return EBNFScriptCreator::Or( {EBNFScriptCreator::Concat( {left_bracket, start_separator, additional_rule_name, EBNFScriptCreator::Repeat( EBNFScriptCreator::Concat({mid_separator, additional_rule_name}), 0, spec.max_items == -1 ? -1 : static_cast(spec.max_items - 1) ), end_separator, right_bracket} ), empty_part} ); } else { return EBNFScriptCreator::Concat( {left_bracket, start_separator, additional_rule_name, EBNFScriptCreator::Repeat( EBNFScriptCreator::Concat({mid_separator, additional_rule_name}), static_cast(spec.min_items - 1), spec.max_items == -1 ? -1 : static_cast(spec.max_items - 1) ), end_separator, right_bracket} ); } } else { std::vector prefix_part; for (size_t i = 0; i < item_rule_names.size(); ++i) { if (i > 0) { prefix_part.push_back(mid_separator); } prefix_part.push_back(item_rule_names[i]); } auto prefix_part_str = EBNFScriptCreator::Concat(prefix_part); if (!spec.allow_additional_items) { return EBNFScriptCreator::Concat( {left_bracket, start_separator, prefix_part_str, end_separator, right_bracket} ); } else { int64_t min_items = std::max( static_cast(0), spec.min_items - static_cast(item_rule_names.size()) ); return EBNFScriptCreator::Concat( {left_bracket, start_separator, prefix_part_str, EBNFScriptCreator::Repeat( EBNFScriptCreator::Concat({mid_separator, additional_rule_name}), static_cast(min_items), spec.max_items == -1 ? -1 : static_cast(spec.max_items - static_cast(item_rule_names.size())) ), end_separator, right_bracket} ); } } } std::string JSONSchemaConverter::FormatPropertyKey(const std::string& key) { return "\"" + JSONStrToPrintableStr(picojson::value(key).serialize()) + "\""; } std::string JSONSchemaConverter::FormatProperty( const std::string& key, const std::string& value_rule, const std::string& rule_name, int64_t idx ) { return FormatPropertyKey(key) + " " + colon_pattern_ + " " + value_rule; } std::string JSONSchemaConverter::FormatOtherProperty( const std::string& key_pattern, const std::string& value_rule, const std::string& rule_name, const std::string& rule_name_suffix ) { return key_pattern + " " + colon_pattern_ + " " + value_rule; } std::string JSONSchemaConverter::GetPropertyWithNumberConstraints( const std::string& pattern, int min_properties, int max_properties, int already_repeated_times ) { if (max_properties != -1 && max_properties == already_repeated_times) { return "\"\""; } int lower = std::max(0, min_properties - already_repeated_times); int upper = max_properties == -1 ? -1 : std::max(-1, max_properties - already_repeated_times); if (lower == 0 && upper == -1) { return "(" + pattern + ")*"; } else if (lower == 0 && upper == 1) { return "(" + pattern + ")?"; } else if (lower == 1 && upper == 1) { return pattern; } else { return "(" + pattern + "){" + std::to_string(lower) + "," + (upper == -1 ? "" : std::to_string(upper)) + "} "; } } std::string JSONSchemaConverter::GetAnyOrderRuleForProperties( const std::vector& properties, const std::unordered_set& required, const SchemaSpecPtr& additional, const std::string& rule_name, const std::string& additional_suffix, int min_properties, int max_properties, const std::string& additional_prop_pattern_override ) { std::string first_sep = NextSeparator(); std::string mid_sep = NextSeparator(); std::string last_sep = NextSeparator(true); // Build one "item" alternation over every property (any required/optional key) plus any // additional/pattern key; any_order does not care which key goes where. std::vector item_patterns; for (size_t idx = 0; idx < properties.size(); ++idx) { const auto& prop = properties[idx]; std::string value_rule = CreateRule(prop.schema, rule_name + "_prop_" + std::to_string(idx)); item_patterns.push_back(FormatProperty(prop.name, value_rule, rule_name, idx)); } if (additional != nullptr) { if (!additional_prop_pattern_override.empty()) { item_patterns.push_back(additional_prop_pattern_override); } else { std::string add_value_rule = CreateRule(additional, rule_name + "_" + additional_suffix); item_patterns.push_back(FormatOtherProperty( GetKeyPatternExcluding(properties, rule_name), add_value_rule, rule_name, additional_suffix )); } } std::string item_body; for (size_t i = 0; i < item_patterns.size(); ++i) { if (i != 0) { item_body += " | "; } item_body += item_patterns[i]; } std::string item_rule = ebnf_script_creator_.AddRule(rule_name + "_item", item_body); // Repeat `item` between n = max(minProperties, #required) and m = maxProperties times; only the // count is constrained, not which keys appear. int min_count = std::max(min_properties, static_cast(required.size())); std::string content = item_rule + " " + GetPropertyWithNumberConstraints(mid_sep + " " + item_rule, min_count, max_properties, 1); return first_sep + " (" + content + ") " + last_sep; } std::string JSONSchemaConverter::GetPartialRuleForProperties( const std::vector& properties, const std::unordered_set& required, const SchemaSpecPtr& additional, const std::string& rule_name, const std::string& additional_suffix, int min_properties, int max_properties, const std::string& additional_prop_pattern_override ) { if (max_properties == 0) { return ""; } if (any_order_) { return GetAnyOrderRuleForProperties( properties, required, additional, rule_name, additional_suffix, min_properties, max_properties, additional_prop_pattern_override ); } std::string first_sep = NextSeparator(); std::string mid_sep = NextSeparator(); std::string last_sep = NextSeparator(true); std::string res = ""; std::vector prop_patterns; for (size_t idx = 0; idx < properties.size(); ++idx) { const auto& prop = properties[idx]; std::string value_rule = CreateRule(prop.schema, rule_name + "_prop_" + std::to_string(idx)); prop_patterns.push_back(FormatProperty(prop.name, value_rule, rule_name, idx)); } if (min_properties == 0 && max_properties == -1) { // Case 1: No property number constraints std::vector rule_names(properties.size(), ""); std::vector is_required(properties.size(), false); bool allow_additional = additional != nullptr; // Construct the last rule std::string additional_prop_pattern; if (allow_additional) { if (!additional_prop_pattern_override.empty()) { additional_prop_pattern = additional_prop_pattern_override; } else { std::string add_value_rule = CreateRule(additional, rule_name + "_" + additional_suffix); additional_prop_pattern = FormatOtherProperty( GetKeyPatternExcluding(properties, rule_name), add_value_rule, rule_name, additional_suffix ); } std::string last_rule_body = "(" + mid_sep + " " + additional_prop_pattern + ")*"; std::string last_rule_name = rule_name + "_part_" + std::to_string(static_cast(properties.size()) - 1); last_rule_name = ebnf_script_creator_.AddRule(last_rule_name, last_rule_body); rule_names.back() = last_rule_name; } else { rule_names.back() = "\"\""; } // Construct 0~(len(properties) - 2) rules for (int i = static_cast(properties.size()) - 2; i >= 0; --i) { const std::string& prop_pattern = prop_patterns[i + 1]; const std::string& last_rule_name = rule_names[i + 1]; std::string cur_rule_body = mid_sep + " " + prop_pattern + " " + last_rule_name; if (!required.count(properties[i + 1].name)) { cur_rule_body = last_rule_name + " | " + cur_rule_body; } else { is_required[i + 1] = true; } std::string cur_rule_name = rule_name + "_part_" + std::to_string(i); cur_rule_name = ebnf_script_creator_.AddRule(cur_rule_name, cur_rule_body); rule_names[i] = cur_rule_name; } if (required.count(properties[0].name)) { is_required[0] = true; } // Construct the root rule for (size_t i = 0; i < properties.size(); ++i) { if (i != 0) { res += " | "; } res += "(" + prop_patterns[i] + " " + rule_names[i] + ")"; if (is_required[i]) { break; } } if (allow_additional && required.empty()) { res += " | " + additional_prop_pattern + " " + rule_names.back(); } res = first_sep + " (" + res + ") " + last_sep; } else if (max_properties == -1) { // Case 2: With constraint on the lower bound of the properties number const int properties_size = static_cast(properties.size()); std::vector> rule_names(properties_size, std::vector()); std::vector key_matched_min(properties_size, 0); std::vector is_required(properties_size, false); bool allow_additional = additional != nullptr; std::string additional_prop_pattern; if (allow_additional) { if (!additional_prop_pattern_override.empty()) { additional_prop_pattern = additional_prop_pattern_override; } else { std::string add_value_rule = CreateRule(additional, rule_name + "_" + additional_suffix); additional_prop_pattern = FormatOtherProperty( GetKeyPatternExcluding(properties, rule_name), add_value_rule, rule_name, additional_suffix ); } } // Get the range of matched properties for each rule bool get_first_required = required.count(properties[0].name); key_matched_min[0] = 1; for (int i = 1; i < properties_size; ++i) { if (required.count(properties[i].name)) { is_required[i] = true; key_matched_min[i] = key_matched_min[i - 1] + 1; } else { key_matched_min[i] = key_matched_min[i - 1]; } if (!get_first_required) { key_matched_min[i] = 1; } if (is_required[i]) { get_first_required = true; } } if (required.count(properties[0].name)) { is_required[0] = true; } if (allow_additional) { key_matched_min.back() = std::max(1, key_matched_min.back()); } else { key_matched_min.back() = std::max(min_properties, key_matched_min.back()); } for (int i = properties_size - 2; i >= 0; --i) { key_matched_min[i] = std::max(key_matched_min[i], key_matched_min[i + 1] - 1); } // Construct the last rule if (allow_additional) { for (int matched = key_matched_min.back(); matched <= properties_size; ++matched) { std::string last_rule_body = GetPropertyWithNumberConstraints( mid_sep + " " + additional_prop_pattern, min_properties, max_properties, matched ); std::string last_rule_name = rule_name + "_part_" + std::to_string(properties_size - 1) + "_" + std::to_string(matched); last_rule_name = ebnf_script_creator_.AddRule(last_rule_name, last_rule_body); rule_names.back().push_back(last_rule_name); } } else { for (int matched = key_matched_min.back(); matched <= properties_size; ++matched) { rule_names.back().push_back("\"\""); } } // Construct 0~(len(properties) - 2) rules for (int i = properties_size - 2; i >= 0; --i) { const std::string& prop_pattern = prop_patterns[i + 1]; for (int matched = key_matched_min[i]; matched <= i + 1; ++matched) { std::string cur_rule_body; if (is_required[i + 1] || matched == key_matched_min[i + 1] - 1) { cur_rule_body = mid_sep + " " + prop_pattern + " " + rule_names[i + 1][matched + 1 - key_matched_min[i + 1]]; } else { cur_rule_body = rule_names[i + 1][matched - key_matched_min[i + 1]] + " | " + mid_sep + " " + prop_pattern + " " + rule_names[i + 1][matched - key_matched_min[i + 1] + 1]; } std::string cur_rule_name = rule_name + "_part_" + std::to_string(i) + "_" + std::to_string(matched); cur_rule_name = ebnf_script_creator_.AddRule(cur_rule_name, cur_rule_body); rule_names[i].push_back(cur_rule_name); } } // Construct root rule bool is_first = true; for (int i = 0; i < properties_size; ++i) { if (key_matched_min[i] > 1) { break; } if (!is_first) { res += " | "; } else { is_first = false; } res += "(" + prop_patterns[i] + " " + rule_names[i][1 - key_matched_min[i]] + ")"; if (is_required[i]) { break; } } if (allow_additional && required.empty()) { if (!is_first) { res += " | "; } res += "(" + additional_prop_pattern + " " + GetPropertyWithNumberConstraints( mid_sep + " " + additional_prop_pattern, min_properties, max_properties, 1 ) + ")"; } res = first_sep + " (" + res + ") " + last_sep; } else { // Case 3: With constraints on both lower & upper bound of the properties number const int properties_size = static_cast(properties.size()); std::vector> rule_names(properties_size, std::vector()); std::vector key_matched_min(properties_size, 0); std::vector key_matched_max(properties_size, properties_size); std::vector is_required(properties_size, false); bool allow_additional = additional != nullptr; std::string additional_prop_pattern; if (allow_additional) { if (!additional_prop_pattern_override.empty()) { additional_prop_pattern = additional_prop_pattern_override; } else { std::string add_value_rule = CreateRule(additional, rule_name + "_" + additional_suffix); additional_prop_pattern = FormatOtherProperty( GetKeyPatternExcluding(properties, rule_name), add_value_rule, rule_name, additional_suffix ); } } // Get the range of matched properties for each rule bool get_first_required = required.count(properties[0].name); key_matched_min[0] = 1; key_matched_max[0] = 1; for (int i = 1; i < properties_size; ++i) { if (required.count(properties[i].name)) { is_required[i] = true; key_matched_min[i] = key_matched_min[i - 1] + 1; } else { key_matched_min[i] = key_matched_min[i - 1]; } if (!get_first_required) { key_matched_min[i] = 1; } key_matched_max[i] = key_matched_max[i - 1] + 1; if (is_required[i]) { get_first_required = true; } } if (required.count(properties[0].name)) { is_required[0] = true; } if (allow_additional) { key_matched_min.back() = std::max(1, key_matched_min.back()); key_matched_max.back() = std::min(max_properties, key_matched_max.back()); } else { key_matched_min.back() = std::max(min_properties, key_matched_min.back()); key_matched_max.back() = std::min(max_properties, key_matched_max.back()); } for (int i = properties_size - 2; i >= 0; --i) { key_matched_min[i] = std::max(key_matched_min[i], key_matched_min[i + 1] - 1); if (is_required[i + 1]) { key_matched_max[i] = std::min(key_matched_max[i], key_matched_max[i + 1] - 1); } else { key_matched_max[i] = std::min(key_matched_max[i], key_matched_max[i + 1]); } } // Construct the last rule if (allow_additional) { for (int matched = key_matched_min.back(); matched <= key_matched_max.back(); ++matched) { std::string last_rule_body = GetPropertyWithNumberConstraints( mid_sep + " " + additional_prop_pattern, min_properties, max_properties, matched ); std::string last_rule_name = rule_name + "_part_" + std::to_string(properties_size - 1) + "_" + std::to_string(matched); last_rule_name = ebnf_script_creator_.AddRule(last_rule_name, last_rule_body); rule_names.back().push_back(last_rule_name); } } else { for (int matched = key_matched_min.back(); matched <= key_matched_max.back(); ++matched) { rule_names.back().push_back("\"\""); } } // Construct 0~(len(properties) - 2) rules for (int i = properties_size - 2; i >= 0; --i) { const std::string& prop_pattern = prop_patterns[i + 1]; for (int matched = key_matched_min[i]; matched <= key_matched_max[i]; ++matched) { std::string cur_rule_body; if (matched == key_matched_max[i + 1]) { cur_rule_body = rule_names[i + 1][matched - key_matched_min[i + 1]]; } else if (is_required[i + 1] || matched == key_matched_min[i + 1] - 1) { cur_rule_body = mid_sep + " " + prop_pattern + " " + rule_names[i + 1][matched + 1 - key_matched_min[i + 1]]; } else { cur_rule_body = rule_names[i + 1][matched - key_matched_min[i + 1]] + " | " + mid_sep + " " + prop_pattern + " " + rule_names[i + 1][matched - key_matched_min[i + 1] + 1]; } std::string cur_rule_name = rule_name + "_part_" + std::to_string(i) + "_" + std::to_string(matched); cur_rule_name = ebnf_script_creator_.AddRule(cur_rule_name, cur_rule_body); rule_names[i].push_back(cur_rule_name); } } // Construct root rule bool is_first = true; for (int i = 0; i < properties_size; ++i) { if (key_matched_max[i] < key_matched_min[i]) { continue; } if (key_matched_min[i] > 1) { break; } if (!is_first) { res += " | "; } else { is_first = false; } res += "(" + prop_patterns[i] + " " + rule_names[i][1 - key_matched_min[i]] + ")"; if (is_required[i]) { break; } } if (allow_additional && required.empty()) { if (!is_first) { res += " | "; } res += "(" + additional_prop_pattern + " " + GetPropertyWithNumberConstraints( mid_sep + " " + additional_prop_pattern, min_properties, max_properties, 1 ) + ")"; } res = first_sep + " (" + res + ") " + last_sep; } return res; } std::string JSONSchemaConverter::GenerateObject( const ObjectSpec& spec, const std::string& rule_name, bool need_braces ) { std::string result = ""; if (need_braces) { result += "\"{\""; } bool could_be_empty = false; // Determine additional property handling std::string additional_suffix = ""; SchemaSpecPtr additional_property; if (spec.allow_additional_properties && spec.additional_properties_schema) { additional_suffix = "addl"; additional_property = spec.additional_properties_schema; } else if (spec.allow_unevaluated_properties && spec.unevaluated_properties_schema) { additional_suffix = "uneval"; additional_property = spec.unevaluated_properties_schema; } else if (spec.allow_additional_properties || spec.allow_unevaluated_properties) { additional_suffix = "addl"; additional_property = SchemaSpec::Make(AnySpec{}, "", "any"); } indent_manager_.StartIndent(); if (!spec.properties.empty() && (!spec.pattern_properties.empty() || spec.property_names)) { // Case 1a: properties coexist with patternProperties and/or propertyNames. // Use GetPartialRuleForProperties for named properties, and build // patternProperties/propertyNames as the additional property pattern override. SchemaSpecPtr effective_additional = additional_property; std::string effective_suffix = additional_suffix; std::string pp_override = ""; if (!spec.pattern_properties.empty()) { // Build patternProperties as additional property alternatives std::string pp_body = ""; for (size_t i = 0; i < spec.pattern_properties.size(); ++i) { const auto& pp = spec.pattern_properties[i]; std::string value = CreateRule(pp.schema, rule_name + "_pp_" + std::to_string(i)); std::string pp_single = "\"\\\"\"" + RegexToEBNF(pp.pattern, false) + "\"\\\"\" " + colon_pattern_ + " " + value; if (i != 0) pp_body += " | "; pp_body += pp_single; } // Merge with existing additionalProperties if present if (effective_additional) { std::string add_value_rule = CreateRule(effective_additional, rule_name + "_" + effective_suffix); std::string add_prop = FormatOtherProperty(GetKeyPattern(), add_value_rule, rule_name, effective_suffix); pp_body += " | " + add_prop; } // Wrap in parentheses to ensure correct EBNF precedence when | is present pp_override = "(" + pp_body + ")"; if (!effective_additional) { effective_additional = SchemaSpec::Make(AnySpec{}, "", "any"); } effective_suffix = "pp"; } else if (spec.property_names && effective_additional) { // propertyNames constrains keys of additional properties. // Only apply when additional properties are allowed — when additionalProperties // is false, no extra keys beyond named properties should be permitted. auto key_pattern = CreateRule(spec.property_names, rule_name + "_name"); std::string val_rule = CreateRule(effective_additional, rule_name + "_" + effective_suffix); pp_override = key_pattern + " " + colon_pattern_ + " " + val_rule; effective_suffix = "pn"; } result += " " + GetPartialRuleForProperties( spec.properties, spec.required, effective_additional, rule_name, effective_suffix, spec.min_properties, spec.max_properties, pp_override ); could_be_empty = spec.required.empty() && spec.min_properties == 0; } else if (!spec.pattern_properties.empty() || spec.property_names) { // Case 1b: patternProperties or propertyNames without named properties (original logic) std::string beg_seq = NextSeparator(); std::string property_rule_body = "("; if (spec.max_properties != 0) { if (!spec.pattern_properties.empty()) { for (size_t i = 0; i < spec.pattern_properties.size(); ++i) { const auto& pp = spec.pattern_properties[i]; std::string value = CreateRule(pp.schema, rule_name + "_prop_" + std::to_string(i)); std::string property_pattern = "\"\\\"\"" + RegexToEBNF(pp.pattern, false) + "\"\\\"\" " + colon_pattern_ + " " + value; if (i != 0) { property_rule_body += " | "; } property_rule_body += "(" + beg_seq + " " + property_pattern + ")"; } property_rule_body += ")"; } else { auto key_pattern = CreateRule(spec.property_names, rule_name + "_name"); property_rule_body += beg_seq + " " + key_pattern + " " + colon_pattern_ + " " + GetBasicAnyRuleName() + ")"; } auto prop_rule_name = ebnf_script_creator_.AllocateRuleName(rule_name + "_prop"); ebnf_script_creator_.AddRuleWithAllocatedName(prop_rule_name, property_rule_body); result += " " + prop_rule_name + " " + GetPropertyWithNumberConstraints( NextSeparator() + " " + prop_rule_name, spec.min_properties, spec.max_properties, 1 ) + NextSeparator(true); could_be_empty = spec.min_properties == 0; } } else if (!spec.properties.empty()) { // Case 2: properties defined (no patternProperties/propertyNames) result += " " + GetPartialRuleForProperties( spec.properties, spec.required, additional_property, rule_name, additional_suffix, spec.min_properties, spec.max_properties ); could_be_empty = spec.required.empty() && spec.min_properties == 0; } else if (additional_property) { // Case 3: no properties defined, additional properties allowed if (spec.max_properties != 0) { std::string add_value_rule = CreateRule(additional_property, rule_name + "_" + additional_suffix); std::string other_property_pattern = FormatOtherProperty(GetKeyPattern(), add_value_rule, rule_name, additional_suffix); result += " " + NextSeparator() + " " + other_property_pattern + " "; result += GetPropertyWithNumberConstraints( NextSeparator() + " " + other_property_pattern, spec.min_properties, spec.max_properties, 1 ) + " " + NextSeparator(true); } could_be_empty = spec.min_properties == 0; } else { // Case 4: no properties, no additional properties, no pattern properties // The object is unconditionally empty. could_be_empty = true; } indent_manager_.EndIndent(); if (need_braces) { result += " \"}\""; } if (could_be_empty) { std::string whitespace_part = GetWhitespacePattern(); auto rest = need_braces ? "\"{\" " + std::string(any_whitespace_ ? whitespace_part + " " : "") + "\"}\"" : std::string(any_whitespace_ ? whitespace_part : ""); if (result == "\"{\" \"}\"" || result == "") { result = rest; } else { result = "(" + result + ") | " + rest; } } if (result.empty()) { return "\"\""; } return result; } std::string JSONSchemaConverter::GenerateAny(const AnySpec& spec, const std::string& rule_name) { return kBasicNumber + " | " + kBasicString + " | " + kBasicBoolean + " | " + kBasicNull + " | " + kBasicArray + " | " + kBasicObject; } std::string JSONSchemaConverter::GenerateConst( const ConstSpec& spec, const std::string& rule_name ) { return "\"" + JSONStrToPrintableStr(spec.json_value) + "\""; } std::string JSONSchemaConverter::GenerateEnum(const EnumSpec& spec, const std::string& rule_name) { XGRAMMAR_DCHECK(!spec.json_values.empty()) << "GenerateEnum called with empty enum spec for rule: " << rule_name; std::string result = ""; for (size_t i = 0; i < spec.json_values.size(); ++i) { if (i != 0) { result += " | "; } result += "(\"" + JSONStrToPrintableStr(spec.json_values[i]) + "\")"; } return result; } std::string JSONSchemaConverter::GenerateRef(const RefSpec& spec, const std::string& rule_name) { // First check if we have a direct URI mapping (for circular references) if (uri_to_rule_name_.count(spec.uri)) { return uri_to_rule_name_[spec.uri]; } if (!ref_resolver_) { XGRAMMAR_LOG(FATAL) << "Ref resolver not set; cannot resolve $ref: " << spec.uri; } // Derive rule name from URI path (like original URIToRule) so that the same // $ref always gets the same rule name, and allocate before resolving to prevent // dead recursion when the ref target contains a ref back. std::string rule_name_hint = "ref"; if (spec.uri.size() >= 2 && spec.uri[0] == '#' && spec.uri[1] == '/') { std::string new_rule_name_prefix; std::stringstream ss(spec.uri.substr(2)); std::string part; while (std::getline(ss, part, '/')) { if (!part.empty()) { if (!new_rule_name_prefix.empty()) { new_rule_name_prefix += "_"; } for (char c : part) { if (std::isalpha(static_cast(c)) || c == '_' || c == '-' || c == '.') { new_rule_name_prefix += c; } } } } if (!new_rule_name_prefix.empty()) { rule_name_hint = std::move(new_rule_name_prefix); } } std::string allocated_rule_name = ebnf_script_creator_.AllocateRuleName(rule_name_hint); uri_to_rule_name_[spec.uri] = allocated_rule_name; SchemaSpecPtr resolved = ref_resolver_(spec.uri, allocated_rule_name); std::string rule_body = GenerateFromSpec(resolved, allocated_rule_name); ebnf_script_creator_.AddRuleWithAllocatedName(allocated_rule_name, rule_body); if (!resolved->cache_key.empty()) { AddCache(resolved->cache_key, allocated_rule_name); } return allocated_rule_name; } std::string JSONSchemaConverter::GenerateAnyOf( const AnyOfSpec& spec, const std::string& rule_name ) { std::string result = ""; for (size_t i = 0; i < spec.options.size(); ++i) { if (i != 0) { result += " | "; } result += CreateRule(spec.options[i], rule_name + "_case_" + std::to_string(i)); } return result; } std::string JSONSchemaConverter::GenerateAllOf( const AllOfSpec& spec, const std::string& rule_name ) { if (spec.schemas.size() == 1) { return GenerateFromSpec(spec.schemas[0], rule_name + "_case_0"); } XGRAMMAR_LOG(WARNING) << "Support for allOf with multiple options is still ongoing"; return GenerateFromSpec(SchemaSpec::Make(AnySpec{}, "", "any"), rule_name); } std::string JSONSchemaConverter::GenerateTypeArray( const TypeArraySpec& spec, const std::string& rule_name ) { std::string result = ""; for (size_t i = 0; i < spec.type_schemas.size(); ++i) { if (i != 0) { result += " | "; } result += CreateRule(spec.type_schemas[i], rule_name + "_type_" + std::to_string(i)); } return result; } // ==================== Static Helper Methods ==================== std::optional JSONSchemaConverter::JSONFormatToRegexPattern(const std::string& format ) { static const auto regex_map = []() -> std::unordered_map { std::unordered_map m; std::string atext = "[\\w!#$%&'*+/=?^`{|}~-]"; std::string dot_string = "(" + atext + "+(\\." + atext + "+)*)"; std::string quoted_string = "\\\\\"(\\\\[\\x20-\\x7E]|[\\x20\\x21\\x23-\\x5B\\x5D-\\x7E])*\\\\\""; std::string domain = "([A-Za-z0-9]([\\-A-Za-z0-9]*[A-Za-z0-9])?)((\\.[A-Za-z0-9][\\-A-Za-z0-9]*[A-Za-z0-9])*" ")"; m["email"] = "^(" + dot_string + "|" + quoted_string + ")@" + domain + "$"; m["date"] = "^(\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2]\\d|3[01]))$"; m["time"] = "^([01]\\d|2[0-3]):[0-5]\\d:([0-5]\\d|60)(\\.\\d+)?(Z|[+-]([01]\\d|2[0-3]):[0-5]\\d)$"; m["date-time"] = "^(\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2]\\d|3[01]))T([01]\\d|2[0-3]):[0-5]\\d:([0-5]\\d|60)(" "\\.\\d+)?(Z|[+-]([01]\\d|2[0-3]):[0-5]\\d)$"; m["duration"] = "^P((\\d+D|\\d+M(\\d+D)?|\\d+Y(\\d+M(\\d+D)?)?)(T(\\d+S|\\d+M(\\d+S)?|\\d+H(\\d+M(\\d+" "S)?" ")?))?|T(\\d+S|\\d+M(\\d+S)?|\\d+H(\\d+M(\\d+S)?)?)|\\d+W)$"; std::string decbyte = "(25[0-5]|2[0-4]\\d|[0-1]?\\d?\\d)"; m["ipv4"] = "^(" + decbyte + "\\.){3}" + decbyte + "$"; m["ipv6"] = "(" "([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|" "([0-9a-fA-F]{1,4}:){1,7}:|" "([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|" "([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|" "([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|" "([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|" "([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|" "[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|" ":((:[0-9a-fA-F]{1,4}){1,7}|:)|" "::(ffff(:0{1,4}){0,1}:){0,1}" "((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}" "(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|" "([0-9a-fA-F]{1,4}:){1,4}:" "((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}" "(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])" ")"; m["hostname"] = "^([a-z0-9]([a-z0-9-]*[a-z0-9])?)(\\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$"; m["uuid"] = "^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$"; std::string schema_pat = "[a-zA-Z][a-zA-Z+\\.-]*"; std::string pchar = "([\\w\\.~!$&'()*+,;=:@-]|%[0-9A-Fa-f][0-9A-Fa-f])"; std::string query_fragment_char = "([\\w\\.~!$&'()*+,;=:@/\\?-]|%[0-9A-Fa-f][0-9A-Fa-f])*"; std::string query = "(\\?" + query_fragment_char + ")?"; std::string fragment = "(#" + query_fragment_char + ")?"; std::string path_abempty = "(/" + pchar + "*)*"; std::string path_absolute_rootless_empty = "/?(" + pchar + "+(/" + pchar + "*)*)?"; std::string userinfo = "([\\w\\.~!$&'()*+,;=:-]|%[0-9A-Fa-f][0-9A-Fa-f])*"; std::string host = "([\\w\\.~!$&'()*+,;=-]|%[0-9A-Fa-f][0-9A-Fa-f])*"; std::string authority = "(" + userinfo + "@)?" + host + "(:\\d*)?"; std::string hier_part = "(//" + authority + path_abempty + "|" + path_absolute_rootless_empty + ")"; m["uri"] = "^" + schema_pat + ":" + hier_part + query + fragment + "$"; pchar = "([\\w\\.~!$&'()*+,;=:@-]|%[0-9A-Fa-f][0-9A-Fa-f])"; query_fragment_char = "([\\w\\.~!$&'()*+,;=:@/\\?-]|%[0-9A-Fa-f][0-9A-Fa-f])*"; query = "(\\?" + query_fragment_char + ")?"; fragment = "(#" + query_fragment_char + ")?"; path_abempty = "(/" + pchar + "*)*"; std::string path_absolute = "/(" + pchar + "+(/" + pchar + "*)*)?"; std::string segment_nz_nc = "([\\w\\.~!$&'()*+,;=@-]|%[0-9A-Fa-f][0-9A-Fa-f])+"; std::string path_noscheme = segment_nz_nc + "(/" + pchar + "*)*"; userinfo = "([\\w\\.~!$&'()*+,;=:-]|%[0-9A-Fa-f][0-9A-Fa-f])*"; host = "([\\w\\.~!$&'()*+,;=-]|%[0-9A-Fa-f][0-9A-Fa-f])*"; authority = "(" + userinfo + "@)?" + host + "(:\\d*)?"; std::string relative_part = "(//" + authority + path_abempty + "|" + path_absolute + "|" + path_noscheme + ")?"; m["uri-reference"] = "^" + relative_part + query + fragment + "$"; std::string literals = "([\\x21\\x23-\\x24\\x26\\x28-\\x3B\\x3D\\x3F-\\x5B\\x5D\\x5F\\x61-\\x7A\\x7E]" "|%[0-9A-Fa-f][0-9A-Fa-f])"; std::string op = "[+#\\./;\\?&=,!@|]"; std::string varchar = "(\\w|%[0-9A-Fa-f][0-9A-Fa-f])"; std::string varname = varchar + "(\\.?" + varchar + ")*"; std::string varspec = varname + "(:[1-9]\\d?\\d?\\d?|\\*)?"; std::string variable_list = varspec + "(," + varspec + ")*"; std::string expression = "\\{(" + op + ")?" + variable_list + "\\}"; m["uri-template"] = "^(" + literals + "|" + expression + ")*$"; m["json-pointer"] = "^(/([\\x00-\\x2E]|[\\x30-\\x7D]|[\\x7F-\\U0010FFFF]|~[01])*)*$"; m["relative-json-pointer"] = "^(0|[1-9][0-9]*)(#|(/([\\x00-\\x2E]|[\\x30-\\x7D]|[\\x7F-\\U0010FFFF]|~[01])*)*)$"; return m; }(); auto it = regex_map.find(format); if (it == regex_map.end()) { return std::nullopt; } return it->second; } std::string JSONSchemaConverter::JSONStrToPrintableStr(const std::string& json_str) { static const std::vector> kReplaceMapping = { {"\\", "\\\\"}, {"\"", "\\\""} }; std::string result = json_str; for (const auto& [k, v] : kReplaceMapping) { size_t pos = 0; while ((pos = result.find(k, pos)) != std::string::npos) { result.replace(pos, k.length(), v); pos += v.length(); } } return result; } bool JSONSchemaConverter::StringSpecKey::operator==(const StringSpecKey& other) const { return pattern == other.pattern && min_length == other.min_length && max_length == other.max_length && wrapper == other.wrapper; } size_t JSONSchemaConverter::StringSpecKeyHash::operator()(const StringSpecKey& key) const { return HashCombine( std::hash()(key.pattern), key.min_length, key.max_length, std::hash()(key.wrapper.first), std::hash()(key.wrapper.second) ); } // ==================== Range Regex Generation ==================== // Stateless utility that turns a numeric range into an anchored regex matching // exactly the JSON integers / numbers inside it. Every method is static; the // class exists only to group the helpers and keep the internal ones private. class NumberGenerator { public: // Anchored regex matching every integer x with start <= x <= end. Either bound // may be std::nullopt for an open side; an empty range yields "^()$". Bounds // span the whole int64 range (|INT64_MIN| is handled without negation overflow). static std::string IntegerRangeRegex(std::optional start, std::optional end); // Anchored regex matching every number in the range, written with up to // `precision` fraction digits. `exclusive_start` / `exclusive_end` exclude the // boundary value itself (turning >= / <= into > / <). Either bound may be // std::nullopt for an open side; an empty range yields "^()$". static std::string FloatRangeRegex( std::optional start, std::optional end, int precision, bool exclusive_start, bool exclusive_end ); private: // Regex alternatives for the fraction digits following a decimal point. struct FracPatternSet { // Each pattern matches a non-empty fraction digit string. std::vector parts; // Whether having no fraction digits at all also satisfies the bound. bool include_empty = false; }; // --- Regex fragment primitives --- static std::string DigitClass(char lo, char hi); // one digit in [lo, hi] (or \d) static std::string ExactDigits(int k); // exactly k free digits: \d{k} static std::string FreeDigits(int max_count); // 0..max_count free digits: \d{0,n} static std::string OptionalZeros(int max_count); // 0..max_count zeros: 0{0,n} static std::string SomeZeros(int max_count); // 1..max_count zeros: 0{1,n} static bool AllChar(const std::string& s, char c); // --- Integer range (operate on non-negative decimal magnitude strings) --- static std::string AbsDigits(int64_t v); static int CompareDigitStr(const std::string& a, const std::string& b); static std::vector IntSameLen(const std::string& a, const std::string& b); static std::vector NumberPatternsStr(const std::string& lo, const std::string& hi); static std::string SubRangeRegexStr(const std::string& lo, const std::string& hi); static std::vector AtLeastPositivePatternsStr(const std::string& v_str); // --- Float range --- static std::string FormatFloat(double value, int precision); // Snaps a non-negative bound to the precision grid in the direction that keeps // the range sound: a lower bound rounds up, an upper bound rounds down, so no // out-of-range value is ever admitted. Returns the canonical grid string and, // via strict_out, whether the boundary value must still be excluded. static std::string RoundBoundToGrid( double value, int precision, bool is_lower, bool strict_in, bool* strict_out ); // Adds (inc) or subtracts (!inc) one grid step (10^-precision) to a canonical // non-negative decimal string, returning the canonical result. static std::string AdjustGrid(const std::string& s, int precision, bool inc); static void SplitDecimal(const std::string& s, std::string* int_part, std::string* frac_part); static int CompareDecimal( const std::string& int_a, const std::string& frac_a, const std::string& int_b, const std::string& frac_b ); static std::string StripAnchors(const std::string& regex); static int64_t ParseIntCapped(const std::string& digits); static FracPatternSet FracGreaterPatterns(const std::string& s, bool strict, int max_len); static FracPatternSet FracLessPatterns(const std::string& s, bool strict, int max_len); static FracPatternSet FracBetweenPatterns( const std::string& a, bool strict_a, const std::string& b, bool strict_b, int max_len ); static std::vector PositiveRangeParts( const std::string& low, bool strict_low, const std::optional& high, bool strict_high, int precision ); }; // Helpers for integer range regex generation. They operate purely on // fixed-length decimal digit strings (suffixes may carry leading zeros), so the // patterns are correct by construction regardless of digit position. // A regex fragment matching a single digit in [lo, hi]. std::string NumberGenerator::DigitClass(char lo, char hi) { if (lo == hi) { return std::string(1, lo); } if (lo == '0' && hi == '9') { return "\\d"; } return "[" + std::string(1, lo) + "-" + std::string(1, hi) + "]"; } // A regex fragment matching k free digits (each 0-9). Empty when k <= 0. std::string NumberGenerator::ExactDigits(int k) { if (k <= 0) { return ""; } if (k == 1) { return "\\d"; } return "\\d{" + std::to_string(k) + "}"; } bool NumberGenerator::AllChar(const std::string& s, char c) { return std::all_of(s.begin(), s.end(), [c](char ch) { return ch == c; }); } // Patterns matching every equal-length digit string t with // value(a) <= value(t) <= value(b). Requires a.size() == b.size() and // value(a) <= value(b). Partitions t by its first digit: // * first digit == a[0]: the suffix must be >= a's suffix (<= 99..9); // * first digit strictly between a[0] and b[0]: the suffix is unconstrained; // * first digit == b[0]: the suffix must be <= b's suffix (>= 00..0). // The partition is exact and non-overlapping, so the union is sound and // complete for [a, b]. std::vector NumberGenerator::IntSameLen(const std::string& a, const std::string& b) { int n = static_cast(a.size()); if (a == b) { return {a}; } if (n == 1) { return {DigitClass(a[0], b[0])}; } if (a[0] == b[0]) { std::vector res; for (auto& p : IntSameLen(a.substr(1), b.substr(1))) { res.push_back(std::string(1, a[0]) + p); } return res; } // a[0] < b[0] std::string a_suf = a.substr(1); std::string b_suf = b.substr(1); if (AllChar(a_suf, '0') && AllChar(b_suf, '9')) { // The whole suffix space is free: collapse to one box pattern. if (a[0] == '0' && b[0] == '9') { return {ExactDigits(n)}; } return {DigitClass(a[0], b[0]) + ExactDigits(n - 1)}; } std::vector res; std::string nines(n - 1, '9'); std::string zeros(n - 1, '0'); for (auto& p : IntSameLen(a_suf, nines)) { res.push_back(std::string(1, a[0]) + p); } if (b[0] - a[0] >= 2) { res.push_back( DigitClass(static_cast(a[0] + 1), static_cast(b[0] - 1)) + ExactDigits(n - 1) ); } for (auto& p : IntSameLen(zeros, b_suf)) { res.push_back(std::string(1, b[0]) + p); } return res; } // Compares two non-negative decimal magnitude strings (no leading zeros except // "0") by value. int NumberGenerator::CompareDigitStr(const std::string& a, const std::string& b) { if (a.size() != b.size()) { return a.size() < b.size() ? -1 : 1; } if (a < b) { return -1; } return a > b ? 1 : 0; } // Patterns matching every integer whose magnitude has value in [lo, hi], where // lo and hi are non-negative decimal magnitude strings (no leading zeros except // "0"). An empty range (value(lo) > value(hi)) yields no patterns. Operating on // strings keeps the whole int64 range representable, including // |INT64_MIN| = 9223372036854775808, which does not fit in int64. std::vector NumberGenerator::NumberPatternsStr( const std::string& lo, const std::string& hi ) { std::vector patterns; if (CompareDigitStr(lo, hi) > 0) { return patterns; } int lo_len = static_cast(lo.size()); int hi_len = static_cast(hi.size()); // Split [lo, hi] by digit length; each length yields a same-length segment // handled exactly by IntSameLen. for (int len = lo_len; len <= hi_len; ++len) { std::string a_str = (len == lo_len) ? lo : ("1" + std::string(len - 1, '0')); std::string b_str = (len == hi_len) ? hi : std::string(len, '9'); for (auto& p : IntSameLen(a_str, b_str)) { patterns.push_back(p); } } return patterns; } // Joins NumberPatternsStr alternatives into a parenthesised regex group. std::string NumberGenerator::SubRangeRegexStr(const std::string& lo, const std::string& hi) { std::vector patterns = NumberPatternsStr(lo, hi); std::string joined; for (size_t i = 0; i < patterns.size(); ++i) { if (i > 0) { joined += "|"; } joined += patterns[i]; } return "(" + joined + ")"; } // Patterns matching every integer in [value(v_str), +infinity) for v_str a // positive magnitude string (no leading zeros). Same-length values come from // IntSameLen(v_str, 99..9); strictly longer values are any non-zero-led number. std::vector NumberGenerator::AtLeastPositivePatternsStr(const std::string& v_str) { int len = static_cast(v_str.size()); std::vector res = IntSameLen(v_str, std::string(len, '9')); res.push_back("[1-9]\\d{" + std::to_string(len) + ",}"); return res; } // The magnitude (absolute value) of v as a decimal string. Derived from the // signed text rather than by negating v, so INT64_MIN is handled correctly. std::string NumberGenerator::AbsDigits(int64_t v) { std::string s = std::to_string(v); return (!s.empty() && s[0] == '-') ? s.substr(1) : s; } std::string NumberGenerator::IntegerRangeRegex( std::optional start, std::optional end ) { std::vector parts; std::ostringstream result; if (!start && !end) { return "^-?\\d+$"; } if (start && !end) { if (start.value() <= 0) { if (start.value() < 0) { // Negatives in [start, -1] are the magnitudes [1, |start|], negated. parts.push_back("-" + SubRangeRegexStr("1", AbsDigits(start.value()))); } parts.push_back("0"); parts.push_back("[1-9]\\d*"); } else { // x >= start with start > 0: same-length values >= start, plus every // value with strictly more digits. for (auto& p : AtLeastPositivePatternsStr(std::to_string(start.value()))) { parts.push_back(p); } } } if (!start && end) { if (end.value() >= 0) { parts.push_back("-[1-9]\\d*"); parts.push_back("0"); if (end.value() > 0) { parts.push_back(SubRangeRegexStr("1", std::to_string(end.value()))); } } else { // x <= end with end < 0: x = -a where a >= |end| > 0, so negate every // pattern for the range [|end|, +infinity). for (auto& p : AtLeastPositivePatternsStr(AbsDigits(end.value()))) { parts.push_back("-" + p); } } } if (start && end) { int64_t range_start = start.value(); int64_t range_end = end.value(); if (range_start > range_end) { return "^()$"; } if (range_start < 0) { int64_t neg_start = range_start; int64_t neg_end = std::min(static_cast(-1), range_end); // Negatives in [neg_start, neg_end] are the magnitudes // [|neg_end|, |neg_start|], negated. parts.push_back("-" + SubRangeRegexStr(AbsDigits(neg_end), AbsDigits(neg_start))); } if (range_start <= 0 && range_end >= 0) { parts.push_back("0"); } if (range_end > 0) { int64_t pos_start = std::max(static_cast(1), range_start); parts.push_back(SubRangeRegexStr(std::to_string(pos_start), std::to_string(range_end))); } } result << "^("; for (size_t i = 0; i < parts.size(); ++i) { if (i > 0) { result << "|"; } result << parts[i]; } result << ")$"; return result.str(); } std::string NumberGenerator::FormatFloat(double value, int precision) { // Casting a double outside [INT64_MIN, INT64_MAX] (or NaN/Inf) to int64_t is // undefined behavior, so range-check before the integer fast path. 2^63 == // 9223372036854775808.0 is exactly representable and one past INT64_MAX, so the // upper comparison must be strict. if (value >= -9223372036854775808.0 && value < 9223372036854775808.0 && value == static_cast(value)) { return std::to_string(static_cast(value)); } std::ostringstream oss; oss << std::fixed << std::setprecision(precision) << value; std::string result = oss.str(); size_t decimalPos = result.find('.'); if (decimalPos != std::string::npos) { size_t lastNonZero = result.find_last_not_of('0'); if (lastNonZero != std::string::npos && lastNonZero > decimalPos) { result.erase(lastNonZero + 1); } else if (lastNonZero == decimalPos) { result.erase(decimalPos); } } return result; } std::string NumberGenerator::AdjustGrid(const std::string& s, int precision, bool inc) { std::string int_part, frac_part; SplitDecimal(s, &int_part, &frac_part); // Build the scaled-integer numerator (value * 10^precision) as a digit string. // Callers only pass FormatFloat output (<= precision fraction digits); guard // the count so a longer string can never wrap the unsigned append count. frac_part.append(std::max(0, precision - static_cast(frac_part.size())), '0'); std::string num = int_part + frac_part; if (inc) { int i = static_cast(num.size()) - 1; for (; i >= 0 && num[i] == '9'; --i) { num[i] = '0'; } if (i < 0) { num.insert(num.begin(), '1'); } else { num[i]++; } } else { int i = static_cast(num.size()) - 1; for (; i >= 0 && num[i] == '0'; --i) { num[i] = '9'; } if (i < 0) { // Underflow below zero; clamp to zero (does not occur for the bounds the // float pipeline feeds in, which are all >= one grid step when decremented). num.assign(num.size(), '0'); } else { num[i]--; } } // Re-split into integer and `precision`-digit fraction, then canonicalize. while (static_cast(num.size()) <= precision) { num.insert(num.begin(), '0'); } std::string new_int = num.substr(0, num.size() - precision); std::string new_frac = num.substr(num.size() - precision); size_t nz = new_int.find_first_not_of('0'); new_int = (nz == std::string::npos) ? "0" : new_int.substr(nz); size_t lnz = new_frac.find_last_not_of('0'); new_frac = (lnz == std::string::npos) ? "" : new_frac.substr(0, lnz + 1); return new_frac.empty() ? new_int : new_int + "." + new_frac; } std::string NumberGenerator::RoundBoundToGrid( double value, int precision, bool is_lower, bool strict_in, bool* strict_out ) { // FormatFloat rounds to the nearest grid point; if that lands exactly on the // bound, keep the original strictness. Otherwise step to the grid point just // inside the range so no out-of-range value is admitted, and the boundary is // now strictly interior, so it becomes inclusive. std::string r = FormatFloat(value, precision); double rv = std::stod(r); if (rv == value) { *strict_out = strict_in; return r; } *strict_out = false; if (is_lower && rv < value) { // Rounded below a lower bound: move up to the smallest grid point >= value. r = AdjustGrid(r, precision, /*inc=*/true); } else if (!is_lower && rv > value) { // Rounded above an upper bound: move down to the largest grid point <= value. r = AdjustGrid(r, precision, /*inc=*/false); } return r; } // Helpers for GenerateFloatRangeRegex. Fraction patterns operate on the // digit string after the decimal point, compared against a canonical bound // fraction (canonical: produced by FormatFloat, so no trailing zeros). // Matches 0 to max_count free digits. std::string NumberGenerator::FreeDigits(int max_count) { if (max_count <= 0) { return ""; } return "\\d{0," + std::to_string(max_count) + "}"; } // Matches 0 to max_count zeros. std::string NumberGenerator::OptionalZeros(int max_count) { if (max_count <= 0) { return ""; } return "0{0," + std::to_string(max_count) + "}"; } // Matches 1 to max_count zeros. std::string NumberGenerator::SomeZeros(int max_count) { return "0{1," + std::to_string(max_count) + "}"; } // Patterns for fraction strings t (1 <= |t| <= max_len) whose value 0.t is // greater than 0.s (or equal when !strict). |s| <= max_len. NumberGenerator::FracPatternSet NumberGenerator::FracGreaterPatterns( const std::string& s, bool strict, int max_len ) { FracPatternSet result; int n = static_cast(s.size()); // t agrees with s up to position i, then has a larger digit for (int i = 0; i < n; ++i) { if (s[i] < '9') { result.parts.push_back( s.substr(0, i) + DigitClass(s[i] + 1, '9') + FreeDigits(max_len - i - 1) ); } } // t extends s with a nonzero digit (after optional zeros) for (int k = 0; n + k + 1 <= max_len; ++k) { result.parts.push_back(s + std::string(k, '0') + "[1-9]" + FreeDigits(max_len - n - k - 1)); } if (!strict) { // t has the same value as s: s plus optional trailing zeros if (n > 0) { result.parts.push_back(s + OptionalZeros(max_len - n)); } else { result.include_empty = true; if (max_len >= 1) { result.parts.push_back(SomeZeros(max_len)); } } } return result; } // Patterns for fraction strings t (1 <= |t| <= max_len) whose value 0.t is // less than 0.s (or equal when !strict). |s| <= max_len. NumberGenerator::FracPatternSet NumberGenerator::FracLessPatterns( const std::string& s, bool strict, int max_len ) { FracPatternSet result; int n = static_cast(s.size()); // t agrees with s up to position i, then has a smaller digit for (int i = 0; i < n; ++i) { if (s[i] > '0') { result.parts.push_back( s.substr(0, i) + DigitClass('0', s[i] - 1) + FreeDigits(max_len - i - 1) ); } } // t is a proper prefix of s plus optional trailing zeros: strictly smaller, // since the remaining digits of s contain a nonzero one for (int i = 0; i < n; ++i) { if (i == 0) { if (max_len >= 1) { result.parts.push_back(SomeZeros(max_len)); } } else { result.parts.push_back(s.substr(0, i) + OptionalZeros(max_len - i)); } } if (!strict) { // t has the same value as s if (n > 0) { result.parts.push_back(s + OptionalZeros(max_len - n)); } else if (max_len >= 1) { result.parts.push_back(SomeZeros(max_len)); } } result.include_empty = n > 0 || !strict; return result; } // Patterns for fraction strings t whose value 0.t lies between 0.a and 0.b. // Requires value(0.a) < value(0.b) and b non-empty. NumberGenerator::FracPatternSet NumberGenerator::FracBetweenPatterns( const std::string& a, bool strict_a, const std::string& b, bool strict_b, int max_len ) { FracPatternSet result; // Longest common prefix of b and zero-padded a. Always stops before |b|: // value(0.a) < value(0.b) implies b is not a prefix of padded a. int common_len = 0; while (common_len < static_cast(b.size()) && (common_len < static_cast(a.size()) ? a[common_len] : '0') == b[common_len]) { ++common_len; } std::string common = b.substr(0, common_len); char digit_a = common_len < static_cast(a.size()) ? a[common_len] : '0'; char digit_b = b[common_len]; // a digit strictly between the bounds' digits, then anything if (digit_b - digit_a >= 2) { result.parts.push_back( common + DigitClass(digit_a + 1, digit_b - 1) + FreeDigits(max_len - common_len - 1) ); } // lower boundary: t continues with digit_a, the rest must exceed a's suffix if (common_len < static_cast(a.size())) { FracPatternSet sub_lower = FracGreaterPatterns(a.substr(common_len + 1), strict_a, max_len - common_len - 1); for (auto& part : sub_lower.parts) { result.parts.push_back(common + digit_a + std::move(part)); } if (sub_lower.include_empty) { result.parts.push_back(common + std::string(1, digit_a)); } } else { // a's value equals value(0.common): only nonzero extensions of // common + digit_a ('0') are strictly greater FracPatternSet sub_lower = FracGreaterPatterns("", true, max_len - common_len - 1); for (auto& part : sub_lower.parts) { result.parts.push_back(common + digit_a + std::move(part)); } if (!strict_a) { // t has the same value as a if (!a.empty()) { result.parts.push_back(a + OptionalZeros(max_len - static_cast(a.size()))); } else { result.include_empty = true; if (max_len >= 1) { result.parts.push_back(SomeZeros(max_len)); } } } } // upper boundary: t continues with digit_b, the rest must stay below b's suffix FracPatternSet sub_upper = FracLessPatterns(b.substr(common_len + 1), strict_b, max_len - common_len - 1); for (auto& part : sub_upper.parts) { result.parts.push_back(common + digit_b + std::move(part)); } if (sub_upper.include_empty) { result.parts.push_back(common + std::string(1, digit_b)); } return result; } // Splits a canonical decimal string from FormatFloat ("12" or "12.34") into // integer and fraction parts. void NumberGenerator::SplitDecimal( const std::string& s, std::string* int_part, std::string* frac_part ) { size_t dot = s.find('.'); if (dot == std::string::npos) { *int_part = s; frac_part->clear(); } else { *int_part = s.substr(0, dot); *frac_part = s.substr(dot + 1); } } // Compares the values of two canonical non-negative decimals. int NumberGenerator::CompareDecimal( const std::string& int_a, const std::string& frac_a, const std::string& int_b, const std::string& frac_b ) { if (int_a.size() != int_b.size()) { return int_a.size() < int_b.size() ? -1 : 1; } if (int_a != int_b) { return int_a < int_b ? -1 : 1; } size_t max_frac = std::max(frac_a.size(), frac_b.size()); for (size_t i = 0; i < max_frac; ++i) { char da = i < frac_a.size() ? frac_a[i] : '0'; char db = i < frac_b.size() ? frac_b[i] : '0'; if (da != db) { return da < db ? -1 : 1; } } return 0; } // Strips the ^( )$ anchors added by IntegerRangeRegex, keeping the group. std::string NumberGenerator::StripAnchors(const std::string& regex) { return regex.substr(1, regex.size() - 2); } int64_t NumberGenerator::ParseIntCapped(const std::string& digits) { // `digits` is a canonical non-negative integer string (no leading zeros). // Parse it exactly when it fits in int64; clamp to INT64_MAX otherwise (such // magnitudes are beyond practical float bounds and double integer precision). static const std::string kMaxInt64 = std::to_string(std::numeric_limits::max()); if (digits.size() > kMaxInt64.size() || (digits.size() == kMaxInt64.size() && digits > kMaxInt64)) { return std::numeric_limits::max(); } return std::stoll(digits); } // Patterns for unsigned decimals (integer part plus optional fraction of up // to `precision` digits) within the given bounds. `low` is required and // non-negative; `high` is optional. Patterns for the value 0 are never // produced: when low's value is 0 the bound is treated as strict, and the // caller emits the zero pattern itself. std::vector NumberGenerator::PositiveRangeParts( const std::string& low, bool strict_low, const std::optional& high, bool strict_high, int precision ) { std::vector parts; std::string int_low, frac_low; SplitDecimal(low, &int_low, &frac_low); if (int_low == "0" && frac_low.empty()) { strict_low = true; } int64_t int_low_value = ParseIntCapped(int_low); std::string opt_any_frac = "(\\.\\d{1," + std::to_string(precision) + "})?"; auto add_with_int_part = [&](const std::string& int_part, const FracPatternSet& set) { for (const auto& part : set.parts) { parts.push_back(int_part + "\\." + part); } if (set.include_empty) { parts.push_back(int_part); } }; if (!high.has_value()) { add_with_int_part(int_low, FracGreaterPatterns(frac_low, strict_low, precision)); // Guard the +1 against int64 overflow (int_low_value may be clamped to // INT64_MAX for very large bounds). if (int_low_value < std::numeric_limits::max()) { parts.push_back( StripAnchors(IntegerRangeRegex(int_low_value + 1, std::nullopt)) + opt_any_frac ); } return parts; } std::string int_high, frac_high; SplitDecimal(*high, &int_high, &frac_high); int64_t int_high_value = ParseIntCapped(int_high); int cmp = CompareDecimal(int_low, frac_low, int_high, frac_high); if (cmp > 0 || (cmp == 0 && (strict_low || strict_high))) { return parts; } if (cmp == 0) { // single representable value, with optional redundant trailing zeros if (frac_low.empty()) { parts.push_back(int_low + "(\\." + SomeZeros(precision) + ")?"); } else { parts.push_back( int_low + "\\." + frac_low + OptionalZeros(precision - static_cast(frac_low.size())) ); } return parts; } if (int_low == int_high) { add_with_int_part( int_low, FracBetweenPatterns(frac_low, strict_low, frac_high, strict_high, precision) ); } else { add_with_int_part(int_low, FracGreaterPatterns(frac_low, strict_low, precision)); if (int_high_value - int_low_value >= 2) { parts.push_back( StripAnchors(IntegerRangeRegex(int_low_value + 1, int_high_value - 1)) + opt_any_frac ); } add_with_int_part(int_high, FracLessPatterns(frac_high, strict_high, precision)); } return parts; } std::string NumberGenerator::FloatRangeRegex( std::optional start, std::optional end, int precision, bool exclusive_start, bool exclusive_end ) { if (start && end) { if (start.value() > end.value() || (start.value() == end.value() && (exclusive_start || exclusive_end))) { return "^()$"; } } if (!start && !end) { return "^-?\\d+(\\.\\d{1," + std::to_string(precision) + "})?$"; } std::vector parts; // Negative values: x is in [start, end] iff -x is in [-end, -start], so the // positive-range patterns are reused on the negated bounds and prefixed // with '-'. bool negatives_in_range = !start.has_value() || start.value() < 0; if (negatives_in_range) { std::string low = "0"; bool strict_low = true; if (end.has_value() && end.value() < 0) { low = RoundBoundToGrid(-end.value(), precision, /*is_lower=*/true, exclusive_end, &strict_low); } std::optional high; bool strict_high = false; if (start.has_value()) { high = RoundBoundToGrid( -start.value(), precision, /*is_lower=*/false, exclusive_start, &strict_high ); } for (auto& part : PositiveRangeParts(low, strict_low, high, strict_high, precision)) { parts.push_back("-" + std::move(part)); } } bool zero_allowed = (!start.has_value() || start.value() < 0 || (start.value() == 0 && !exclusive_start)) && (!end.has_value() || end.value() > 0 || (end.value() == 0 && !exclusive_end)); if (zero_allowed) { parts.push_back("0(\\." + SomeZeros(precision) + ")?"); // Negative zero written with an all-zero fraction ("-0.0".."-0.000000") also // denotes 0. PositiveRangeParts never emits magnitude 0, so add these forms // explicitly when the range covers the negative side. if (negatives_in_range) { parts.push_back("-0(\\." + SomeZeros(precision) + ")"); } } // Positive values if (!end.has_value() || end.value() > 0) { std::string low = "0"; bool strict_low = true; if (start.has_value() && start.value() > 0) { low = RoundBoundToGrid( start.value(), precision, /*is_lower=*/true, exclusive_start, &strict_low ); } std::optional high; bool strict_high = false; if (end.has_value()) { high = RoundBoundToGrid(end.value(), precision, /*is_lower=*/false, exclusive_end, &strict_high); } for (auto& part : PositiveRangeParts(low, strict_low, high, strict_high, precision)) { parts.push_back(std::move(part)); } } std::ostringstream result; result << "^("; for (size_t i = 0; i < parts.size(); ++i) { if (i > 0) { result << "|"; } result << parts[i]; } result << ")$"; return result.str(); } std::string JSONSchemaConverter::GenerateRangeRegex( std::optional start, std::optional end ) { return NumberGenerator::IntegerRangeRegex(start, end); } std::string JSONSchemaConverter::GenerateFloatRangeRegex( std::optional start, std::optional end, int precision, bool exclusive_start, bool exclusive_end ) { return NumberGenerator::FloatRangeRegex(start, end, precision, exclusive_start, exclusive_end); } // ==================== Public API Functions ==================== std::string JSONSchemaToEBNF( const std::string& schema, bool any_whitespace, std::optional indent, std::optional> separators, bool strict_mode, std::optional max_whitespace_cnt, JSONFormat json_format, bool any_order ) { picojson::value schema_value; std::string err = picojson::parse(schema_value, schema); XGRAMMAR_CHECK(err.empty()) << "Failed to parse JSON: " << err << ". The JSON string is:" << schema; return JSONSchemaToEBNF( schema_value, any_whitespace, indent, separators, strict_mode, max_whitespace_cnt, json_format, any_order ); } std::string JSONSchemaToEBNF( const picojson::value& schema, bool any_whitespace, std::optional indent, std::optional> separators, bool strict_mode, std::optional max_whitespace_cnt, JSONFormat json_format, bool any_order ) { // Parse JSON Schema to SchemaSpec SchemaParser parser(schema, {strict_mode, json_format}); auto spec_result = parser.Parse(schema, "root"); if (spec_result.IsErr()) { XGRAMMAR_LOG(FATAL) << std::move(spec_result).UnwrapErr().what(); } auto spec = std::move(spec_result).Unwrap(); auto ref_resolver = [&parser](const std::string& uri, const std::string& rule_name_hint) { auto r = parser.ResolveRef(uri, rule_name_hint); if (r.IsErr()) { XGRAMMAR_LOG(FATAL) << std::move(r).UnwrapErr().what(); } return std::move(r).Unwrap(); }; // Create converter based on format switch (json_format) { case JSONFormat::kJSON: { JSONSchemaConverter converter( indent, separators, any_whitespace, max_whitespace_cnt, ref_resolver, any_order ); return converter.Convert(spec); } case JSONFormat::kQwenXML: case JSONFormat::kMiniMaxXML: case JSONFormat::kDeepSeekXML: case JSONFormat::kGlmXML: { XMLToolCallingConverter converter( indent, separators, any_whitespace, max_whitespace_cnt, ref_resolver, json_format, any_order ); return converter.Convert(spec); } default: XGRAMMAR_LOG(FATAL) << "Invalid JSON format: " << static_cast(json_format); } XGRAMMAR_UNREACHABLE(); } // Wrapper functions for testing std::string GenerateRangeRegex(std::optional start, std::optional end) { return JSONSchemaConverter::GenerateRangeRegex(start, end); } std::string GenerateFloatRangeRegex( std::optional start, std::optional end, bool exclusive_start, bool exclusive_end ) { return JSONSchemaConverter::GenerateFloatRangeRegex( start, end, 6, exclusive_start, exclusive_end ); } std::string QwenXMLToolCallingToEBNF(const std::string& schema, bool any_order) { picojson::value json_value; std::string err = picojson::parse(json_value, schema); if (!err.empty()) { XGRAMMAR_LOG(FATAL) << "Failed to parse JSON schema: " << err; } return JSONSchemaToEBNF( json_value, true, std::nullopt, std::nullopt, true, std::nullopt, JSONFormat::kQwenXML, any_order ); } std::string MiniMaxXMLToolCallingToEBNF(const std::string& schema, bool any_order) { picojson::value json_value; std::string err = picojson::parse(json_value, schema); if (!err.empty()) { XGRAMMAR_LOG(FATAL) << "Failed to parse JSON schema: " << err; } return JSONSchemaToEBNF( json_value, true, std::nullopt, std::nullopt, true, std::nullopt, JSONFormat::kMiniMaxXML, any_order ); } std::string DeepSeekXMLToolCallingToEBNF(const std::string& schema, bool any_order) { picojson::value json_value; std::string err = picojson::parse(json_value, schema); if (!err.empty()) { XGRAMMAR_LOG(FATAL) << "Failed to parse JSON schema: " << err; } return JSONSchemaToEBNF( json_value, true, std::nullopt, std::nullopt, true, std::nullopt, JSONFormat::kDeepSeekXML, any_order ); } std::string GlmXMLToolCallingToEBNF(const std::string& schema, bool any_order) { picojson::value json_value; std::string err = picojson::parse(json_value, schema); if (!err.empty()) { XGRAMMAR_LOG(FATAL) << "Failed to parse JSON schema: " << err; } return JSONSchemaToEBNF( json_value, true, std::nullopt, std::nullopt, true, std::nullopt, JSONFormat::kGlmXML, any_order ); } } // namespace xgrammar xgrammar-0.2.3/cpp/json_schema_converter.h000066400000000000000000000476721521764210300206660ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/json_schema_converter.h * \brief Convert a JSON schema string to EBNF grammar string. */ #ifndef XGRAMMAR_JSON_SCHEMA_CONVERTER_H_ #define XGRAMMAR_JSON_SCHEMA_CONVERTER_H_ #include #include #include #include #include #include #include #include #include #include #include "ebnf_script_creator.h" namespace xgrammar { // ==================== SchemaSpec: Intermediate Representation for JSON Schema ==================== // Forward declaration struct SchemaSpec; using SchemaSpecPtr = std::shared_ptr; // Basic Type Specs struct IntegerSpec { std::optional minimum; std::optional maximum; std::optional exclusive_minimum; std::optional exclusive_maximum; std::string ToString() const; }; struct NumberSpec { std::optional minimum; std::optional maximum; std::optional exclusive_minimum; std::optional exclusive_maximum; std::string ToString() const; }; struct StringSpec { std::optional pattern; std::optional format; int min_length = 0; int max_length = -1; // -1 means no limit std::string ToString() const; }; struct BooleanSpec { std::string ToString() const; }; struct NullSpec { std::string ToString() const; }; struct AnySpec { std::string ToString() const; }; // Complex Type Specs struct ArraySpec { std::vector prefix_items; bool allow_additional_items = true; SchemaSpecPtr additional_items; // nullptr means not allowed int64_t min_items = 0; int64_t max_items = -1; // -1 means no limit std::string ToString() const; }; struct ObjectSpec { struct Property { std::string name; SchemaSpecPtr schema; }; struct PatternProperty { std::string pattern; // regex pattern for key SchemaSpecPtr schema; }; std::vector properties; std::vector pattern_properties; std::unordered_set required; bool allow_additional_properties = false; SchemaSpecPtr additional_properties_schema; bool allow_unevaluated_properties = true; SchemaSpecPtr unevaluated_properties_schema; SchemaSpecPtr property_names; int min_properties = 0; int max_properties = -1; // -1 means no limit std::string ToString() const; }; // Composite Type Specs struct ConstSpec { std::string json_value; // JSON serialized value std::string ToString() const; }; struct EnumSpec { std::vector json_values; // JSON serialized values std::string ToString() const; }; struct RefSpec { std::string uri; std::string ToString() const; }; struct AnyOfSpec { std::vector options; std::string ToString() const; }; struct AllOfSpec { std::vector schemas; std::string ToString() const; }; struct TypeArraySpec { // Handle "type": ["string", "integer"] cases std::vector type_schemas; std::string ToString() const; }; // Unified SchemaSpec using SchemaSpecVariant = std::variant< IntegerSpec, NumberSpec, StringSpec, BooleanSpec, NullSpec, ArraySpec, ObjectSpec, AnySpec, ConstSpec, EnumSpec, RefSpec, AnyOfSpec, AllOfSpec, TypeArraySpec>; struct SchemaSpec { SchemaSpecVariant spec; std::string cache_key; // for deduplication std::string rule_name_hint; // suggested rule name std::string ToString() const; // Helper method to create SchemaSpec template static SchemaSpecPtr Make(T&& spec_value, std::string cache_key = "", std::string hint = "") { auto ptr = std::make_shared(); ptr->spec = std::forward(spec_value); ptr->cache_key = std::move(cache_key); ptr->rule_name_hint = std::move(hint); return ptr; } }; // ==================== JSONFormat Enum ==================== enum class JSONFormat : int { kJSON = 0, kQwenXML = 1, kMiniMaxXML = 2, kDeepSeekXML = 3, kGlmXML = 4, }; /*! * \brief Manage the rule generation cache. Wraps key-value cache for schema deduplication. */ class GenerateCacheManager { public: /*! \brief Add a key-value pair to the cache. */ void AddCache(const std::string& key, bool is_inner_layer, const std::string& value) { cache_[{key, is_inner_layer}] = value; } /*! \brief Get cached value by key. Returns std::nullopt if not found. */ std::optional GetCache(const std::string& key, bool is_inner_layer) const { auto it = cache_.find({key, is_inner_layer}); if (it != cache_.end()) { return it->second; } return std::nullopt; } private: std::unordered_map, std::string> cache_; }; /*! * \brief Manage the indent and separator for the generation of EBNF grammar. */ class IndentManager { public: IndentManager( std::optional indent, const std::string& separator, bool any_whitespace, std::optional max_whitespace_cnt ); void StartIndent(); void EndIndent(); std::string StartSeparator(); std::string MiddleSeparator(); std::string EndSeparator(); std::string EmptySeparator(); std::string NextSeparator(bool is_end = false); private: bool any_whitespace_; bool enable_newline_; int64_t indent_; std::string separator_; int64_t total_indent_; std::vector is_first_; std::optional max_whitespace_cnt_; friend class JSONSchemaConverter; }; /*! * \brief Convert SchemaSpec to EBNF grammar string. * * This is the base class for EBNF generation. It generates JSON-format EBNF by default. * Subclasses can override virtual methods to generate different formats (e.g., XML). */ class JSONSchemaConverter { public: using RefResolver = std::function; JSONSchemaConverter( std::optional indent, std::optional> separators, bool any_whitespace, std::optional max_whitespace_cnt, RefResolver ref_resolver = nullptr, bool any_order = false ); virtual ~JSONSchemaConverter() = default; /*! * \brief Convert SchemaSpec to EBNF grammar string. * \param spec The SchemaSpec to convert. * \return The EBNF grammar string. */ std::string Convert(const SchemaSpecPtr& spec); protected: // ==================== Virtual methods for generation ==================== // Subclasses can override these to customize output format virtual std::string GenerateInteger(const IntegerSpec& spec, const std::string& rule_name); virtual std::string GenerateNumber(const NumberSpec& spec, const std::string& rule_name); virtual std::string GenerateString(const StringSpec& spec, const std::string& rule_name); virtual std::string GenerateBoolean(const BooleanSpec& spec, const std::string& rule_name); virtual std::string GenerateNull(const NullSpec& spec, const std::string& rule_name); virtual std::string GenerateArray(const ArraySpec& spec, const std::string& rule_name); virtual std::string GenerateObject( const ObjectSpec& spec, const std::string& rule_name, bool need_brace = true ); virtual std::string GenerateAny(const AnySpec& spec, const std::string& rule_name); virtual std::string GenerateConst(const ConstSpec& spec, const std::string& rule_name); virtual std::string GenerateEnum(const EnumSpec& spec, const std::string& rule_name); virtual std::string GenerateRef(const RefSpec& spec, const std::string& rule_name); virtual std::string GenerateAnyOf(const AnyOfSpec& spec, const std::string& rule_name); virtual std::string GenerateAllOf(const AllOfSpec& spec, const std::string& rule_name); virtual std::string GenerateTypeArray(const TypeArraySpec& spec, const std::string& rule_name); // ==================== Hooks for customization ==================== /*! \brief Format a property key. Override for different formats. */ virtual std::string FormatPropertyKey(const std::string& key); /*! \brief Format a property (key + value). Override for different formats. */ virtual std::string FormatProperty( const std::string& key, const std::string& value_rule, const std::string& rule_name, int64_t idx ); /*! \brief Format an "other" property (additional/unevaluated). Override for different formats. */ virtual std::string FormatOtherProperty( const std::string& key_pattern, const std::string& value_rule, const std::string& rule_name, const std::string& rule_name_suffix ); /*! \brief Get the basic string rule name. Override for different formats. */ virtual std::string GetKeyPattern() const; /*! \brief Get a key pattern that excludes specific property names. */ virtual std::string GetKeyPatternExcluding( const std::vector& properties, const std::string& rule_name ); /*! \brief Get the basic any rule name. Override for different formats. */ virtual std::string GetBasicAnyRuleName() const; /*! \brief Add basic rules for the format. Override for different formats. */ virtual void AddBasicRules(); /*! \brief Add a key-value pair to the generation cache. Override for custom cache behavior. */ virtual void AddCache(const std::string& key, const std::string& value); /*! \brief Get cached value by key. Returns std::nullopt if not found. */ virtual std::optional GetCache(const std::string& key) const; // ==================== Helper methods (for subclasses to use) ==================== /*! \brief Dispatch to the appropriate Generate method based on spec type. */ std::string GenerateFromSpec(const SchemaSpecPtr& spec, const std::string& rule_name_hint); /*! \brief Create a rule and return the rule name (handles caching). */ std::string CreateRule(const SchemaSpecPtr& spec, const std::string& rule_name_hint); /*! \brief Get next separator from indent manager. */ virtual std::string NextSeparator(bool is_end = false); /*! \brief Get whitespace pattern. */ std::string GetWhitespacePattern() const; /*! \brief Helper to create rule with repetition constraints. */ std::string GetPropertyWithNumberConstraints( const std::string& pattern, int min_properties, int max_properties, int already_repeated_times = 0 ); /*! \brief Generate partial rule for object properties. * \param additional_prop_pattern_override When non-empty, used as the additional property * pattern instead of the default GetKeyPattern() : value. This supports patternProperties * and propertyNames constraints on additional keys. */ std::string GetPartialRuleForProperties( const std::vector& properties, const std::unordered_set& required, const SchemaSpecPtr& additional, const std::string& rule_name, const std::string& additional_suffix, int min_properties, int max_properties, const std::string& additional_prop_pattern_override = "" ); /*! \brief Generate the object rule in "any order" mode: an "item" alternation over all property * keys, repeated between max(min_properties, required.size()) and max_properties times. Only the * entry count is bounded, not which keys appear. */ std::string GetAnyOrderRuleForProperties( const std::vector& properties, const std::unordered_set& required, const SchemaSpecPtr& additional, const std::string& rule_name, const std::string& additional_suffix, int min_properties, int max_properties, const std::string& additional_prop_pattern_override = "" ); // ==================== Protected members ==================== EBNFScriptCreator ebnf_script_creator_; IndentManager indent_manager_; std::string colon_pattern_; bool any_whitespace_; std::optional max_whitespace_cnt_; // When true, object properties may appear in any order (see GetAnyOrderRuleForProperties). // Applies to all objects (including nested ones). Default false preserves the fixed-order // behavior. bool any_order_ = false; public: // Basic rule names static const std::string kBasicAny; static const std::string kBasicInteger; static const std::string kBasicNumber; static const std::string kBasicString; static const std::string kBasicBoolean; static const std::string kBasicNull; static const std::string kBasicArray; static const std::string kBasicObject; static const std::string kBasicEscape; static const std::string kBasicStringSub; protected: GenerateCacheManager rule_cache_manager_; private: void AddHelperRules(); std::unordered_map uri_to_rule_name_; // For circular reference handling RefResolver ref_resolver_; // Resolves $ref URI to SchemaSpecPtr at generate time // For string spec deduplication struct StringSpecKey { std::string pattern; int min_length = 0; int max_length = -1; std::pair wrapper; bool operator==(const StringSpecKey& other) const; }; struct StringSpecKeyHash { size_t operator()(const StringSpecKey& key) const; }; std::unordered_map string_spec_cache_; // Helper for integer/number range regex generation static std::string GenerateRangeRegex(std::optional start, std::optional end); static std::string GenerateFloatRangeRegex( std::optional start, std::optional end, int precision = 6, bool exclusive_start = false, bool exclusive_end = false ); // JSON string helpers static std::string JSONStrToPrintableStr(const std::string& json_str); protected: static std::optional JSONFormatToRegexPattern(const std::string& format); // Expose for testing friend std::string GenerateRangeRegex(std::optional start, std::optional end); friend std::string GenerateFloatRangeRegex( std::optional start, std::optional end, bool exclusive_start, bool exclusive_end ); }; // ==================== Public API functions (backward compatible) ==================== /*! * \brief Convert JSON schema string to EBNF grammar string. * \param schema The JSON schema string. * \param any_whitespace Whether to ignore the indentation restrictions, and allow any whitespace. * Default: true. * \param indent The number of spaces for indentation. If set to std::nullopt, the output will be * in one line. Default: 2. * \param separators Two separators used in the schema: comma and colon. Examples: {",", ":"}, * {", ", ": "}. If std::nullopt, the default separators will be used: {",", ": "} when the * indent is not -1, and {", ", ": "} otherwise. This follows the convention in python * json.dumps(). Default: std::nullopt. * \param strict_mode Whether to use strict mode. In strict * mode, the generated grammar will not allow properties and items that is not specified in the * schema. This is equivalent to setting unevaluatedProperties and unevaluatedItems to false. * This helps LLM to generate accurate output in the grammar-guided generation with JSON * schema. Default: true. * \param max_whitespace_cnt The maximum number of whitespace characters for the whitespace * which is used for indentation or JSON elements separation when any_whitespace is True. If * std::nullopt, it means unlimited. Default: std::nullopt. * \param json_format Define the root * format of the object. If it's JSONFormat::kJSON, then it will generate a fully JSON-style * grammar. If it's JSONFormat::kXML, then it will generate a grammar with the root format is * XML-style, while the inner format is JSON-style. Default: JSONFormat::kJSON. * \returns The EBNF grammar string. */ std::string JSONSchemaToEBNF( const std::string& schema, bool any_whitespace = true, std::optional indent = std::nullopt, std::optional> separators = std::nullopt, bool strict_mode = true, std::optional max_whitespace_cnt = std::nullopt, JSONFormat json_format = JSONFormat::kJSON, bool any_order = false ); /*! * \brief Convert JSON schema string to EBNF grammar string. * \param schema The JSON schema object. * \param any_whitespace Whether to ignore the indentation restrictions, and allow any whitespace. * Default: true. * \param indent The number of spaces for indentation. If set to std::nullopt, the output will be * in one line. Default: 2. * \param separators Two separators used in the schema: comma and colon. Examples: {",", ":"}, * {", ", ": "}. If std::nullopt, the default separators will be used: {",", ": "} when the * indent is not -1, and {", ", ": "} otherwise. This follows the convention in python * json.dumps(). Default: std::nullopt. * \param strict_mode Whether to use strict mode. In strict * mode, the generated grammar will not allow properties and items that is not specified in the * schema. This is equivalent to setting unevaluatedProperties and unevaluatedItems to false. * This helps LLM to generate accurate output in the grammar-guided generation with JSON * schema. Default: true. * \param max_whitespace_cnt The maximum number of whitespace characters for the whitespace * which is used for indentation or JSON elements separation when any_whitespace is True. If * std::nullopt, it means unlimited. Default: std::nullopt. * \param json_format Define the root format of the object. If it's JSONFormat::kJSON, * then it will generate a fully JSON-style grammar. If it's JSONFormat::kXML, then it will * generate a grammar with the root format is XML-style, while the inner format is JSON-style. * Default: JSONFormat::kJSON. * \returns The EBNF grammar string. */ std::string JSONSchemaToEBNF( const picojson::value& schema, bool any_whitespace = true, std::optional indent = std::nullopt, std::optional> separators = std::nullopt, bool strict_mode = true, std::optional max_whitespace_cnt = std::nullopt, JSONFormat json_format = JSONFormat::kJSON, bool any_order = false ); /*! * \brief Generate regex pattern for integer/float range. * \param start The start of the range (inclusive). If null assume negative infinity. * \param end The end of the range (inclusive). If null assume infinity. * \returns The regex pattern that matches integers/floats in the given range. */ std::string GenerateRangeRegex(std::optional start, std::optional end); std::string GenerateFloatRangeRegex( std::optional start, std::optional end, bool exclusive_start = false, bool exclusive_end = false ); /*! * \brief Convert a function call to a Grammar. * \param schema The schema of the parameters of the function call. * \return The ebnf-grammar to match the requirements of the schema, and * in Qwen xml style. */ std::string QwenXMLToolCallingToEBNF(const std::string& schema, bool any_order = false); /*! * \brief Convert a function call to a Grammar. * \param schema The schema of the parameters of the function call. * \return The ebnf-grammar to match the requirements of the schema, and * in MiniMax xml style. */ std::string MiniMaxXMLToolCallingToEBNF(const std::string& schema, bool any_order = false); /*! * \brief Convert a function call to a Grammar. * \param schema The schema of the parameters of the function call. * \return The ebnf-grammar to match the requirements of the schema, and * in DeepSeek xml style. */ std::string DeepSeekXMLToolCallingToEBNF(const std::string& schema, bool any_order = false); /*! * \brief Convert a function call to a Grammar. * \param schema The schema of the parameters of the function call. * \return The ebnf-grammar to match the requirements of the schema, and * in GLM xml style (keyvalue). */ std::string GlmXMLToolCallingToEBNF(const std::string& schema, bool any_order = false); } // namespace xgrammar #endif // XGRAMMAR_JSON_SCHEMA_CONVERTER_H_ xgrammar-0.2.3/cpp/json_schema_converter_ext.cc000066400000000000000000000250601521764210300216670ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/json_schema_converter_ext.cc * \brief Implementation of extended format converters. */ #include "json_schema_converter_ext.h" #include #include "json_schema_converter.h" #include "regex_converter.h" namespace xgrammar { // Static constants const std::string XMLToolCallingConverter::kXMLString = "xml_string"; const std::string XMLToolCallingConverter::kXMLAny = "xml_any"; const std::string XMLToolCallingConverter::kXMLObject = "xml_object"; const std::string XMLToolCallingConverter::kXMLVariableName = "xml_variable_name"; const std::unordered_map XMLToolCallingConverter::kKeyWrapperMap = { {JSONFormat::kQwenXML, {"", "", ""}}, {JSONFormat::kMiniMaxXML, {"", "", ""}}, {JSONFormat::kDeepSeekXML, {"<｜DSML｜parameter name=\\\"", "\\\" string=\\\"\" (\"true\" | \"false\") \"\\\">", "", // TODO(Linzhang): we do not validate the string's value, and we accept both. ""}}, {JSONFormat::kGlmXML, {"", "", "", ""}}, }; XMLToolCallingConverter::XMLToolCallingConverter( std::optional indent, std::optional> separators, bool any_whitespace, std::optional max_whitespace_cnt, RefResolver ref_resolver, JSONFormat json_format, bool any_order ) : JSONSchemaConverter( indent, separators, any_whitespace, max_whitespace_cnt, ref_resolver, any_order ), nested_object_level_(0), xml_wrapper_(kKeyWrapperMap.at(json_format)) {} std::string XMLToolCallingConverter::Convert(const SchemaSpecPtr& spec) { nested_object_level_ = 0; AddBasicRules(); std::string root_rule_name = ebnf_script_creator_.AllocateRuleName("root"); std::string root_body = GenerateFromSpec(spec, root_rule_name); ebnf_script_creator_.AddRuleWithAllocatedName(root_rule_name, root_body); return ebnf_script_creator_.GetScript(); } void XMLToolCallingConverter::AddBasicRules() { // First add JSON basic rules. These should be in the inner layer of the XML format. XGRAMMAR_DCHECK(nested_object_level_ == 0); // The nested part, true json format, is at level 2. nested_object_level_ = 2; JSONSchemaConverter::AddBasicRules(); nested_object_level_ = 1; // The outer part, xml format, is at level 1. // Add XML string rule ebnf_script_creator_.AddRule( kXMLString, "TagDispatch(" "loop_after_dispatch=false," "excludes=(\"" + xml_wrapper_.parameter_suffix + "\")" ")" ); constexpr const char* kStringCacheKey = "{\"type\":\"string\"}"; AddCache(kStringCacheKey, kXMLString); // Add XML any rule auto any_spec = SchemaSpec::Make(AnySpec{}, "{}", kXMLAny); std::string any_body = GenerateAny(std::get(any_spec->spec), kXMLAny); ebnf_script_creator_.AddRule(kXMLAny, any_body); AddCache("{}", kXMLAny); // Reset the nested object level to 0, which is the root level. nested_object_level_ = 0; // Add XML object rule constexpr const char* kObjectCacheKey = "{\"type\":\"object\"}"; ObjectSpec obj_spec_val; obj_spec_val.allow_additional_properties = true; obj_spec_val.additional_properties_schema = any_spec; auto obj_spec = SchemaSpec::Make(std::move(obj_spec_val), kObjectCacheKey, kXMLObject); std::string obj_body = GenerateObject(std::get(obj_spec->spec), kXMLObject); ebnf_script_creator_.AddRule(kXMLObject, obj_body); AddCache(kObjectCacheKey, kXMLObject); // Add XML variable name rule std::string var_body = "[a-zA-Z_][a-zA-Z0-9_]*"; ebnf_script_creator_.AddRule(kXMLVariableName, var_body); } std::string XMLToolCallingConverter::GetKeyPattern() const { if (nested_object_level_ <= 1) { return kXMLVariableName; } return kBasicString; } std::string XMLToolCallingConverter::GetBasicAnyRuleName() const { if (nested_object_level_ <= 1) { return kXMLAny; } return kBasicAny; } std::string XMLToolCallingConverter::GetKeyPatternExcluding( const std::vector& properties, const std::string& rule_name ) { if (nested_object_level_ <= 1) { return GetKeyPattern(); } return JSONSchemaConverter::GetKeyPatternExcluding(properties, rule_name); } std::string XMLToolCallingConverter::NextSeparator(bool is_end) { if (nested_object_level_ <= 1) { return GetWhitespacePattern(); } return JSONSchemaConverter::NextSeparator(is_end); } std::string XMLToolCallingConverter::GenerateString( const StringSpec& spec, const std::string& rule_name ) { if (nested_object_level_ <= 1) { // For XML format, use TagDispatch for strings if (!spec.pattern.has_value() && !spec.format.has_value() && spec.min_length == 0 && spec.max_length == -1) { return kXMLString; } if (spec.format.has_value()) { const std::string& format = *spec.format; auto regex_pattern = JSONFormatToRegexPattern(format); if (regex_pattern.has_value()) { std::string converted_regex = RegexToEBNF(regex_pattern.value(), false); return converted_regex; } } // Check for pattern if (spec.pattern.has_value()) { std::string converted_regex = RegexToEBNF(*spec.pattern, false); return converted_regex; } // Check for length constraints if (spec.min_length != 0 || spec.max_length != -1) { std::string char_pattern = "[^]"; std::string repetition; if (spec.max_length == -1) { repetition = "{" + std::to_string(spec.min_length) + ",}"; } else { repetition = "{" + std::to_string(spec.min_length) + "," + std::to_string(spec.max_length) + "}"; } return char_pattern + repetition; } } return JSONSchemaConverter::GenerateString(spec, rule_name); } std::string XMLToolCallingConverter::GenerateAny( const AnySpec& spec, const std::string& rule_name ) { if (nested_object_level_ == 0) { return kXMLObject; } if (nested_object_level_ == 1) { return kXMLString + " | " + kBasicArray + " | " + kBasicObject; } return JSONSchemaConverter::GenerateAny(spec, rule_name); } std::string XMLToolCallingConverter::GenerateArray( const ArraySpec& spec, const std::string& rule_name ) { nested_object_level_++; auto result = JSONSchemaConverter::GenerateArray(spec, rule_name); nested_object_level_--; return result; } std::string XMLToolCallingConverter::GenerateConst( const ConstSpec& spec, const std::string& rule_name ) { if (nested_object_level_ <= 1) { const std::string& val = spec.json_value; if (val.size() >= 2 && val.front() == '"' && val.back() == '"') { return "\"" + val.substr(1, val.size() - 2) + "\""; } return "\"" + val + "\""; } return JSONSchemaConverter::GenerateConst(spec, rule_name); } std::string XMLToolCallingConverter::GenerateEnum( const EnumSpec& spec, const std::string& rule_name ) { XGRAMMAR_DCHECK(!spec.json_values.empty()) << "GenerateEnum called with empty enum spec for rule: " << rule_name; if (nested_object_level_ <= 1) { std::string result; for (size_t i = 0; i < spec.json_values.size(); ++i) { if (i != 0) { result += " | "; } const std::string& val = spec.json_values[i]; if (val.size() >= 2 && val.front() == '"' && val.back() == '"') { result += "(\"" + val.substr(1, val.size() - 2) + "\")"; } else { result += "(\"" + val + "\")"; } } return result; } return JSONSchemaConverter::GenerateEnum(spec, rule_name); } std::string XMLToolCallingConverter::FormatPropertyKey(const std::string& key) { if (nested_object_level_ <= 1) { return "\"" + xml_wrapper_.key_wrapper_prefix + key + xml_wrapper_.key_wrapper_suffix + "\""; } return JSONSchemaConverter::FormatPropertyKey(key); } std::string XMLToolCallingConverter::FormatProperty( const std::string& key, const std::string& value_rule, const std::string& rule_name, int64_t idx ) { if (nested_object_level_ <= 1) { std::string whitespace = GetWhitespacePattern(); if (!xml_wrapper_.value_wrapper_prefix.empty()) { return "\"" + xml_wrapper_.key_wrapper_prefix + key + xml_wrapper_.key_wrapper_suffix + "\" " + whitespace + " \"" + xml_wrapper_.value_wrapper_prefix + "\" " + whitespace + " " + value_rule + " " + whitespace + " \"" + xml_wrapper_.parameter_suffix + "\""; } return "\"" + xml_wrapper_.key_wrapper_prefix + key + xml_wrapper_.key_wrapper_suffix + "\" " + whitespace + " " + value_rule + " " + whitespace + " \"" + xml_wrapper_.parameter_suffix + "\""; } return JSONSchemaConverter::FormatProperty(key, value_rule, rule_name, idx); } std::string XMLToolCallingConverter::FormatOtherProperty( const std::string& key_pattern, const std::string& value_rule, const std::string& rule_name, const std::string& rule_name_suffix ) { if (nested_object_level_ <= 1) { std::string whitespace = GetWhitespacePattern(); if (!xml_wrapper_.value_wrapper_prefix.empty()) { return "\"" + xml_wrapper_.key_wrapper_prefix + "\" " + key_pattern + " \"" + xml_wrapper_.key_wrapper_suffix + "\" " + whitespace + " \"" + xml_wrapper_.value_wrapper_prefix + "\" " + whitespace + " " + value_rule + " " + whitespace + " \"" + xml_wrapper_.parameter_suffix + "\""; } return "\"" + xml_wrapper_.key_wrapper_prefix + "\" " + key_pattern + " \"" + xml_wrapper_.key_wrapper_suffix + "\" " + whitespace + " " + value_rule + " " + whitespace + " \"" + xml_wrapper_.parameter_suffix + "\""; } return JSONSchemaConverter::FormatOtherProperty( key_pattern, value_rule, rule_name, rule_name_suffix ); } std::string XMLToolCallingConverter::GenerateObject( const ObjectSpec& spec, const std::string& rule_name, bool dummy_need_braces ) { nested_object_level_++; bool need_brace = nested_object_level_ > 1; auto result = JSONSchemaConverter::GenerateObject(spec, rule_name, need_brace); nested_object_level_--; return result; } void XMLToolCallingConverter::AddCache(const std::string& key, const std::string& value) { if (key.empty()) { return; } rule_cache_manager_.AddCache(key, nested_object_level_ > 1, value); } std::optional XMLToolCallingConverter::GetCache(const std::string& key) const { if (key.empty()) { return std::nullopt; } return rule_cache_manager_.GetCache(key, nested_object_level_ > 1); } } // namespace xgrammar xgrammar-0.2.3/cpp/json_schema_converter_ext.h000066400000000000000000000066261521764210300215400ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/json_schema_converter_ext.h * \brief Extended format converters for JSON Schema, including XML Tool Calling format. */ #ifndef XGRAMMAR_JSON_SCHEMA_CONVERTER_EXT_H_ #define XGRAMMAR_JSON_SCHEMA_CONVERTER_EXT_H_ #include #include #include "json_schema_converter.h" namespace xgrammar { /*! * \brief Converter for XML Tool Calling format (e.g., Qwen style). * * This converter generates EBNF where: * - The outermost object uses XML format: value * - Inner values use standard JSON format */ class XMLToolCallingConverter : public JSONSchemaConverter { public: XMLToolCallingConverter( std::optional indent, std::optional> separators, bool any_whitespace, std::optional max_whitespace_cnt, RefResolver ref_resolver = nullptr, JSONFormat json_format = JSONFormat::kQwenXML, bool any_order = false ); /*! \brief Convert SchemaSpec to EBNF with XML format for root object. Note that this function is * not thread-safe.*/ std::string Convert(const SchemaSpecPtr& spec); protected: // Override methods for XML format std::string GenerateString(const StringSpec& spec, const std::string& rule_name) override; std::string GenerateObject( const ObjectSpec& spec, const std::string& rule_name, bool dummy_need_braces = false ) override; std::string GenerateAny(const AnySpec& spec, const std::string& rule_name) override; std::string GenerateArray(const ArraySpec& spec, const std::string& rule_name) override; std::string GenerateConst(const ConstSpec& spec, const std::string& rule_name) override; std::string GenerateEnum(const EnumSpec& spec, const std::string& rule_name) override; // Override format hooks std::string FormatPropertyKey(const std::string& key) override; std::string FormatProperty( const std::string& key, const std::string& value_rule, const std::string& rule_name, int64_t idx ) override; std::string FormatOtherProperty( const std::string& key_pattern, const std::string& value_rule, const std::string& rule_name, const std::string& rule_name_suffix ) override; std::string GetKeyPattern() const override; std::string GetBasicAnyRuleName() const override; std::string GetKeyPatternExcluding( const std::vector& properties, const std::string& rule_name ) override; std::string NextSeparator(bool is_end = false) override; void AddBasicRules() override; void AddCache(const std::string& key, const std::string& value) override; std::optional GetCache(const std::string& key) const override; private: // Wrapper strings for XML parameter tags (key prefix/suffix, value prefix, closing suffix) struct XMLWrapper { std::string key_wrapper_prefix; std::string key_wrapper_suffix; std::string value_wrapper_prefix; std::string parameter_suffix; }; static const std::unordered_map kKeyWrapperMap; static const std::string kXMLString; static const std::string kXMLAny; static const std::string kXMLObject; static const std::string kXMLVariableName; // Track if we're at the root object level int nested_object_level_ = 0; const XMLWrapper xml_wrapper_; }; } // namespace xgrammar #endif // XGRAMMAR_JSON_SCHEMA_CONVERTER_EXT_H_ xgrammar-0.2.3/cpp/regex_converter.cc000066400000000000000000000303631521764210300176320ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/regex_converter.cc */ #include "regex_converter.h" #include #include #include #include #include "support/encoding.h" #include "support/logging.h" #include "support/utils.h" namespace xgrammar { /*! * \brief Convert a regex to EBNF. * \details The implementation refers to the regex described in * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Regular_expressions */ class RegexConverter { public: explicit RegexConverter(const std::string& regex) : regex_(regex) { if (!regex.empty()) { regex_codepoints_ = ParseUTF8(regex_.c_str(), false); if (regex_codepoints_[0] == kInvalidUTF8) { XGRAMMAR_LOG(FATAL) << "The regex is not a valid UTF-8 string."; XGRAMMAR_UNREACHABLE(); } } regex_codepoints_.push_back(0); // Add a null terminator } std::string Convert(); private: /** * \brief Add a segment string to the result EBNF string. It especially adds a space if needed * and add_space is true. */ void AddEBNFSegment(const std::string& element); [[noreturn]] void RaiseError(const std::string& message); void RaiseWarning(const std::string& message); std::string HandleCharacterClass(); std::string HandleRepetitionRange(); std::string HandleCharEscape(); std::string HandleEscape(); std::string HandleEscapeInCharClass(); /** * \brief Handle group modifier. The general format is "(?" + modifier + content + ")". E.g. * "(?:abc)" is a non-capturing group. */ void HandleGroupModifier(); std::string regex_; std::vector regex_codepoints_; TCodepoint* start_; TCodepoint* current_; TCodepoint* end_; std::string result_ebnf_; int parenthesis_level_ = 0; }; void RegexConverter::AddEBNFSegment(const std::string& element) { if (!result_ebnf_.empty()) { result_ebnf_ += ' '; } result_ebnf_ += element; } void RegexConverter::RaiseError(const std::string& message) { XGRAMMAR_LOG(FATAL) << "Regex parsing error at position " << current_ - start_ + 1 << ": " << message; XGRAMMAR_UNREACHABLE(); } void RegexConverter::RaiseWarning(const std::string& message) { XGRAMMAR_LOG(WARNING) << "Regex parsing warning at position " << current_ - start_ + 1 << ": " << message; } std::string RegexConverter::HandleCharacterClass() { std::string char_class = "["; ++current_; if (*current_ == ']') { RaiseError("Empty character class is not allowed in regex."); } while (*current_ != ']' && current_ != end_) { if (*current_ == '\\') { char_class += HandleEscapeInCharClass(); } else { char_class += CharToUTF8(*current_); ++current_; } } if (current_ == end_) { RaiseError("Unclosed '['"); } char_class += ']'; ++current_; return char_class; } // {x}: Match exactly x occurrences of the preceding regular expression. // {x,} // {x,y} std::string RegexConverter::HandleRepetitionRange() { std::string result = "{"; ++current_; if (!isdigit(*current_)) { RaiseError("Invalid repetition count."); } while (isdigit(*current_)) { result += static_cast(*current_); ++current_; } if (*current_ != ',' && *current_ != '}') { RaiseError("Invalid repetition count."); } result += static_cast(*current_); ++current_; if (current_[-1] == '}') { // Matches {x} return result; } if (!isdigit(*current_) && *current_ != '}') { RaiseError("Invalid repetition count."); } while (isdigit(*current_)) { result += static_cast(*current_); ++current_; } if (*current_ != '}') { RaiseError("Invalid repetition count."); } result += '}'; ++current_; return result; } std::string RegexConverter::HandleCharEscape() { // clang-format off static const std::unordered_map CUSTOM_ESCAPE_MAP = { {'^', '^'}, {'$', '$'}, {'.', '.'}, {'*', '*'}, {'+', '+'}, {'?', '?'}, {'\\', '\\'}, {'(', '('}, {')', ')'}, {'[', '['}, {']', ']'}, {'{', '{'}, {'}', '}'}, {'|', '|'}, {'/', '/'}, {'-', '-'} }; // clang-format on if (end_ - current_ < 2 || (current_[1] == 'u' && end_ - current_ < 5) || (current_[1] == 'x' && end_ - current_ < 4) || (current_[1] == 'c' && end_ - current_ < 3)) { RaiseError("Escape sequence is not finished."); } auto [codepoint, len] = ParseNextEscaped(current_, CUSTOM_ESCAPE_MAP); if (codepoint != CharHandlingError::kInvalidEscape) { current_ += len; return EscapeString(codepoint); } else if (current_[1] == 'u' && current_[2] == '{') { current_ += 3; int len = 0; TCodepoint value = 0; while (HexCharToInt(current_[len]) != -1 && len <= 6) { value = value * 16 + HexCharToInt(current_[len]); ++len; } if (len == 0 || len > 6 || current_[len] != '}') { RaiseError("Invalid Unicode escape sequence."); } current_ += len + 1; return EscapeString(value); } else if (current_[1] == 'c') { current_ += 2; if (!std::isalpha(*current_)) { RaiseError("Invalid control character escape sequence."); } ++current_; return EscapeString((*(current_ - 1)) % 32); } else { RaiseWarning( "Escape sequence '\\" + EscapeString(current_[1]) + "' is not recognized. The character itself will be matched" ); current_ += 2; return EscapeString(current_[-1]); } } std::string RegexConverter::HandleEscapeInCharClass() { if (end_ - current_ < 2) { RaiseError("Escape sequence is not finished."); } if (current_[1] == 'd') { current_ += 2; return "0-9"; } else if (current_[1] == 'D') { current_ += 2; return R"(\x00-\x2F\x3A-\U0010FFFF)"; } else if (current_[1] == 'w') { current_ += 2; return "a-zA-Z0-9_"; } else if (current_[1] == 'W') { current_ += 2; return R"(\x00-\x2F\x3A-\x40\x5B-\x5E\x60\x7B-\U0010FFFF)"; } else if (current_[1] == 's') { current_ += 2; return R"(\f\n\r\t\v\u0020\u00a0)"; } else if (current_[1] == 'S') { current_ += 2; return R"(\x00-\x08\x0E-\x1F\x21-\x9F\xA1-\U0010FFFF)"; } else { auto res = HandleCharEscape(); if (res == "]" || res == "-") { return "\\" + res; } else { return res; } } } std::string RegexConverter::HandleEscape() { // clang-format off static const std::unordered_map CUSTOM_ESCAPE_MAP = { {'^', '^'}, {'$', '$'}, {'.', '.'}, {'*', '*'}, {'+', '+'}, {'?', '?'}, {'\\', '\\'}, {'(', '('}, {')', ')'}, {'[', '['}, {']', ']'}, {'{', '{'}, {'}', '}'}, {'|', '|'}, {'/', '/'} }; // clang-format on if (end_ - current_ < 2) { RaiseError("Escape sequence is not finished."); } if (current_[1] == 'd') { current_ += 2; return "[0-9]"; } else if (current_[1] == 'D') { current_ += 2; return "[^0-9]"; } else if (current_[1] == 'w') { current_ += 2; return "[a-zA-Z0-9_]"; } else if (current_[1] == 'W') { current_ += 2; return "[^a-zA-Z0-9_]"; } else if (current_[1] == 's') { current_ += 2; return R"([\f\n\r\t\v\u0020\u00a0])"; } else if (current_[1] == 'S') { current_ += 2; return R"([^[\f\n\r\t\v\u0020\u00a0])"; } else if ((current_[1] >= '1' && current_[1] <= '9') || current_[1] == 'k') { RaiseError("Backreference is not supported yet."); } else if (current_[1] == 'p' || current_[1] == 'P') { RaiseError("Unicode character class escape sequence is not supported yet."); } else if (current_[1] == 'b' || current_[1] == 'B') { RaiseError("Word boundary is not supported yet."); } else { return "\"" + HandleCharEscape() + "\""; } } void RegexConverter::HandleGroupModifier() { if (current_ == end_) { RaiseError("Group modifier is not finished."); } if (*current_ == ':') { // Non-capturing group. ++current_; } else if (*current_ == '=' || *current_ == '!') { // Positive or negative lookahead. RaiseError("Lookahead is not supported yet."); } else if (*current_ == '<' && current_ + 1 != end_ && (current_[1] == '=' || current_[1] == '!')) { // Positive or negative lookbehind. RaiseError("Lookbehind is not supported yet."); } else if (*current_ == '<') { ++current_; while (current_ != end_ && isalpha(*current_)) { ++current_; } if (current_ == end_ || *current_ != '>') { RaiseError("Invalid named capturing group."); } // Just ignore the named of the group. ++current_; } else { // Group modifier flag. RaiseError("Group modifier flag is not supported yet."); } } std::string RegexConverter::Convert() { start_ = regex_codepoints_.data(); current_ = start_; end_ = start_ + regex_codepoints_.size() - 1; bool is_empty = true; while (current_ != end_) { if (*current_ == '^') { if (current_ != start_) { RaiseWarning( "'^' should be at the start of the regex, but found in the middle. It is ignored." ); } ++current_; } else if (*current_ == '$') { if (current_ != end_ - 1) { RaiseWarning( "'$' should be at the end of the regex, but found in the middle. It is ignored." ); } ++current_; } else if (*current_ == '[') { is_empty = false; AddEBNFSegment(HandleCharacterClass()); } else if (*current_ == '(') { is_empty = false; ++current_; ++parenthesis_level_; AddEBNFSegment("("); if (current_ != end_ && *current_ == '?') { ++current_; HandleGroupModifier(); } } else if (*current_ == ')') { is_empty = false; if (parenthesis_level_ == 0) { RaiseError("Unmatched ')'"); } // Empty alternative before ')' (e.g. "(a|)" or "(a|$)"): emit "" so it isn't a bare '|'. if (!result_ebnf_.empty() && result_ebnf_.back() == '|') { AddEBNFSegment("\"\""); } --parenthesis_level_; AddEBNFSegment(")"); ++current_; } else if (*current_ == '*' || *current_ == '+' || *current_ == '?') { is_empty = false; result_ebnf_ += static_cast(*current_); ++current_; if (current_ != end_ && *current_ == '?') { // Ignore the non-greedy modifier because our grammar handles all repetition numbers // non-deterministically. ++current_; } if (current_ != end_ && (*current_ == '{' || *current_ == '*' || *current_ == '+' || *current_ == '?')) { RaiseError("Two consecutive repetition modifiers are not allowed."); } } else if (*current_ == '{') { is_empty = false; result_ebnf_ += HandleRepetitionRange(); if (current_ != end_ && *current_ == '?') { // Still ignore the non-greedy modifier. ++current_; } if (current_ != end_ && (*current_ == '{' || *current_ == '*' || *current_ == '+' || *current_ == '?')) { RaiseError("Two consecutive repetition modifiers are not allowed."); } } else if (*current_ == '|') { is_empty = false; // Empty alternative before '|': emit "" so there's no bare '|' on the left. // Covers leading ("^$|abc"), consecutive ("a||b") and group-start ("(|a)") cases. if (result_ebnf_.empty() || result_ebnf_.back() == '|' || result_ebnf_.back() == '(') { AddEBNFSegment("\"\""); } AddEBNFSegment("|"); ++current_; } else if (*current_ == '\\') { is_empty = false; AddEBNFSegment(HandleEscape()); } else if (*current_ == '.') { is_empty = false; AddEBNFSegment(R"([\u0000-\U0010FFFF])"); ++current_; } else { is_empty = false; // Non-special characters are matched literally. AddEBNFSegment("\"" + EscapeString(*current_) + "\""); ++current_; } } if (parenthesis_level_ != 0) { RaiseError("The parenthesis is not closed."); } // Trailing empty alternative, e.g. "abc|": emit "" so it doesn't end with a bare '|'. if (!result_ebnf_.empty() && result_ebnf_.back() == '|') { AddEBNFSegment("\"\""); } if (is_empty) { AddEBNFSegment("\"\""); } return result_ebnf_; } std::string RegexToEBNF(const std::string& regex, bool with_rule_name) { RegexConverter converter(regex); if (with_rule_name) { return "root ::= " + converter.Convert() + "\n"; } else { return converter.Convert(); } } } // namespace xgrammar xgrammar-0.2.3/cpp/regex_converter.h000066400000000000000000000007211521764210300174670ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/regex_converter.h * \brief Convert a regex string to EBNF grammar string. */ #ifndef XGRAMMAR_REGEX_CONVERTER_H_ #define XGRAMMAR_REGEX_CONVERTER_H_ #include namespace xgrammar { /*! * \brief Convert a regex string to EBNF grammar string. */ std::string RegexToEBNF(const std::string& regex, bool with_rule_name = true); } // namespace xgrammar #endif // XGRAMMAR_REGEX_CONVERTER_H_ xgrammar-0.2.3/cpp/structural_tag.cc000066400000000000000000002736021521764210300175010ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/structural_tag.cc */ #include "structural_tag.h" #include #include #include #include #include #include #include #include #include #include #include "grammar_builder.h" #include "grammar_functor.h" #include "grammar_impl.h" #include "json_schema_converter.h" #include "support/logging.h" #include "support/recursion_guard.h" #include "support/utils.h" #include "tokenizer_info_impl.h" #include "xgrammar/grammar.h" namespace xgrammar { // Short alias for the error type. using ISTError = InvalidStructuralTagError; // Forward declaration for helpers that convert Format to picojson::value. picojson::value FormatToJSONValue(const Format& format); picojson::value StringVectorToJSONArray(const std::vector& vector) { picojson::array array; array.reserve(vector.size()); for (const auto& string : vector) { array.push_back(picojson::value(string)); } return picojson::value(std::move(array)); } picojson::value FormatVectorToJSONArray(const std::vector& vector) { picojson::array array; array.reserve(vector.size()); for (const auto& format : vector) { array.push_back(xgrammar::FormatToJSONValue(format)); } return picojson::value(std::move(array)); } picojson::value TagVectorToJSONArray(const std::vector& vector) { picojson::array array; array.reserve(vector.size()); for (const auto& tag : vector) { array.push_back(tag.ToJSON()); } return picojson::value(std::move(array)); } picojson::value IntOrStringVectorToJSONArray( const std::vector>& vec ) { picojson::array array; array.reserve(vec.size()); for (const auto& item : vec) { if (std::holds_alternative(item)) { array.push_back(picojson::value(static_cast(std::get(item)))); } else { array.push_back(picojson::value(std::get(item))); } } return picojson::value(std::move(array)); } /******************** Format To JSON ********************/ std::string FormatToJSON(const Format& format) { return FormatToJSONValue(format).serialize(); } picojson::value FormatToJSONValue(const Format& format) { return std::visit([&](auto&& arg) { return arg.ToJSON(); }, format); } picojson::value ConstStringFormat::ToJSON() const { picojson::object obj; obj["type"] = picojson::value(type); obj["value"] = picojson::value(value); return picojson::value(std::move(obj)); } picojson::value JSONSchemaFormat::ToJSON() const { picojson::object obj; obj["type"] = picojson::value(type); picojson::value schema_val; if (picojson::parse(schema_val, json_schema).empty()) { obj["json_schema"] = schema_val; } else { obj["json_schema"] = picojson::value(json_schema); } obj["style"] = picojson::value(style); obj["any_order"] = picojson::value(any_order); return picojson::value(std::move(obj)); } picojson::value GrammarFormat::ToJSON() const { picojson::object obj; obj["type"] = picojson::value(type); obj["grammar"] = picojson::value(grammar); return picojson::value(std::move(obj)); } picojson::value RegexFormat::ToJSON() const { picojson::object obj; obj["type"] = picojson::value(type); obj["pattern"] = picojson::value(pattern); return picojson::value(std::move(obj)); } picojson::value AnyTextFormat::ToJSON() const { picojson::object obj; obj["type"] = picojson::value(type); obj["excludes"] = StringVectorToJSONArray(excludes); obj["detected_end_strs"] = StringVectorToJSONArray(detected_end_strs_); return picojson::value(std::move(obj)); } // These two constructors are defined here rather than inline because instantiating // vector against the still-incomplete Format variant is ill-formed under C++20. SequenceFormat::SequenceFormat(std::vector elements) : elements(std::move(elements)) {} OrFormat::OrFormat(std::vector elements) : elements(std::move(elements)) {} picojson::value SequenceFormat::ToJSON() const { picojson::object obj; obj["type"] = picojson::value(type); obj["elements"] = FormatVectorToJSONArray(elements); return picojson::value(std::move(obj)); } picojson::value OrFormat::ToJSON() const { picojson::object obj; obj["type"] = picojson::value(type); obj["elements"] = FormatVectorToJSONArray(elements); return picojson::value(std::move(obj)); } picojson::value TagFormat::ToJSON() const { picojson::object obj; obj["type"] = picojson::value(type); if (std::holds_alternative(begin)) { obj["begin"] = picojson::value(std::get(begin)); } else { obj["begin"] = std::get(begin).ToJSON(); } if (content) { obj["content"] = FormatToJSONValue(*content); } else { obj["content"] = picojson::value(); } if (std::holds_alternative(end)) { obj["end"] = std::get(end).ToJSON(); } else { const auto& end_strs = std::get>(end); if (end_strs.size() == 1) { obj["end"] = picojson::value(end_strs[0]); } else { obj["end"] = StringVectorToJSONArray(end_strs); } } return picojson::value(std::move(obj)); } picojson::value TokenFormat::ToJSON() const { picojson::object obj; obj["type"] = picojson::value(type); if (std::holds_alternative(token)) { obj["token"] = picojson::value(static_cast(std::get(token))); } else { obj["token"] = picojson::value(std::get(token)); } return picojson::value(std::move(obj)); } picojson::value ExcludeTokenFormat::ToJSON() const { picojson::object obj; obj["type"] = picojson::value(type); obj["exclude_tokens"] = IntOrStringVectorToJSONArray(exclude_tokens); return picojson::value(std::move(obj)); } picojson::value AnyTokensFormat::ToJSON() const { picojson::object obj; obj["type"] = picojson::value(type); obj["exclude_tokens"] = IntOrStringVectorToJSONArray(exclude_tokens); return picojson::value(std::move(obj)); } picojson::value TokenTriggeredTagsFormat::ToJSON() const { picojson::object obj; obj["type"] = picojson::value(type); obj["trigger_tokens"] = IntOrStringVectorToJSONArray(trigger_tokens); obj["tags"] = TagVectorToJSONArray(tags); obj["exclude_tokens"] = IntOrStringVectorToJSONArray(exclude_tokens); obj["at_least_one"] = picojson::value(at_least_one); obj["stop_after_first"] = picojson::value(stop_after_first); return picojson::value(std::move(obj)); } picojson::value TriggeredTagsFormat::ToJSON() const { picojson::object obj; obj["type"] = picojson::value(type); obj["triggers"] = StringVectorToJSONArray(triggers); obj["tags"] = TagVectorToJSONArray(tags); obj["excludes"] = StringVectorToJSONArray(excludes); obj["at_least_one"] = picojson::value(at_least_one); obj["stop_after_first"] = picojson::value(stop_after_first); obj["detected_end_strs"] = StringVectorToJSONArray(detected_end_strs_); return picojson::value(std::move(obj)); } picojson::value TagsWithSeparatorFormat::ToJSON() const { picojson::object obj; obj["type"] = picojson::value(type); obj["tags"] = TagVectorToJSONArray(tags); obj["separator"] = picojson::value(separator); obj["at_least_one"] = picojson::value(at_least_one); obj["stop_after_first"] = picojson::value(stop_after_first); return picojson::value(std::move(obj)); } picojson::value OptionalFormat::ToJSON() const { picojson::object obj; obj["type"] = picojson::value(type); obj["content"] = FormatToJSONValue(*content); return picojson::value(std::move(obj)); } picojson::value PlusFormat::ToJSON() const { picojson::object obj; obj["type"] = picojson::value(type); obj["content"] = FormatToJSONValue(*content); return picojson::value(std::move(obj)); } picojson::value StarFormat::ToJSON() const { picojson::object obj; obj["type"] = picojson::value(type); obj["content"] = FormatToJSONValue(*content); return picojson::value(std::move(obj)); } picojson::value RepeatFormat::ToJSON() const { picojson::object obj; obj["type"] = picojson::value(type); obj["min"] = picojson::value(static_cast(min)); obj["max"] = picojson::value(static_cast(max)); obj["content"] = FormatToJSONValue(*content); return picojson::value(std::move(obj)); } picojson::value DispatchFormat::ToJSON() const { picojson::object obj; obj["type"] = picojson::value(type); picojson::array rules_arr; rules_arr.reserve(rules.size()); for (const auto& pair : rules) { picojson::array pair_arr; pair_arr.push_back(picojson::value(pair.first)); if (pair.second) { pair_arr.push_back(FormatToJSONValue(*pair.second)); } else { pair_arr.push_back(picojson::value()); } rules_arr.push_back(picojson::value(std::move(pair_arr))); } obj["rules"] = picojson::value(std::move(rules_arr)); obj["loop"] = picojson::value(loop); obj["excludes"] = StringVectorToJSONArray(excludes); return picojson::value(std::move(obj)); } picojson::value TokenDispatchFormat::ToJSON() const { picojson::object obj; obj["type"] = picojson::value(type); picojson::array rules_arr; rules_arr.reserve(rules.size()); for (const auto& pair : rules) { picojson::array pair_arr; if (std::holds_alternative(pair.first)) { pair_arr.push_back(picojson::value(static_cast(std::get(pair.first)))); } else { pair_arr.push_back(picojson::value(std::get(pair.first))); } if (pair.second) { pair_arr.push_back(FormatToJSONValue(*pair.second)); } else { pair_arr.push_back(picojson::value()); } rules_arr.push_back(picojson::value(std::move(pair_arr))); } obj["rules"] = picojson::value(std::move(rules_arr)); obj["loop"] = picojson::value(loop); obj["exclude_tokens"] = IntOrStringVectorToJSONArray(exclude_tokens); return picojson::value(std::move(obj)); } /************** StructuralTag Parser **************/ class StructuralTagParser { public: static Result FromJSON(const std::string& json); private: Result ParseStructuralTag(const picojson::value& value); /*! * \brief Parse a Format object from a JSON value. * \param value The JSON value to parse. * \return A Format object if the JSON is valid, otherwise an error message in std::runtime_error. * \note The "type" field is checked in this function, and not checked in the Parse*Format * functions. */ Result ParseFormat(const picojson::value& value); Result ParseConstStringFormat(const picojson::object& value); Result ParseJSONSchemaFormat( const picojson::object& value, std::optional style_override = std::nullopt ); Result ParseAnyTextFormat(const picojson::object& value); Result ParseGrammarFormat(const picojson::object& value); Result ParseRegexFormat(const picojson::object& value); Result ParseSequenceFormat(const picojson::object& value); Result ParseOrFormat(const picojson::object& value); /*! \brief ParseTagFormat with extra check for object and the type field. */ Result ParseTagFormat(const picojson::value& value); Result ParseTagFormat(const picojson::object& value); Result ParseTriggeredTagsFormat(const picojson::object& value); Result ParseTagsWithSeparatorFormat( const picojson::object& value ); Result ParseOptionalFormat(const picojson::object& value); Result ParsePlusFormat(const picojson::object& value); Result ParseStarFormat(const picojson::object& value); Result ParseRepeatFormat(const picojson::object& value); Result ParseTokenFormat(const picojson::object& value); Result ParseExcludeTokenFormat(const picojson::object& value); Result ParseAnyTokensFormat(const picojson::object& value); Result ParseTokenTriggeredTagsFormat( const picojson::object& value ); Result ParseDispatchFormat(const picojson::object& value); Result ParseTokenDispatchFormat(const picojson::object& value); int parse_format_recursion_depth_ = 0; }; Result StructuralTagParser::FromJSON(const std::string& json) { picojson::value value; std::string err = picojson::parse(value, json); if (!err.empty()) { return ResultErr("Failed to parse JSON: " + err); } return Result::Convert( StructuralTagParser().ParseStructuralTag(value) ); } Result StructuralTagParser::ParseStructuralTag(const picojson::value& value ) { if (!value.is()) { return ResultErr("Structural tag must be an object"); } const auto& obj = value.get(); // The type field is optional but must be "structural_tag" if present. if (obj.find("type") != obj.end()) { if (!obj["type"].is() || obj["type"].get() != "structural_tag") { return ResultErr("Structural tag's type must be a string \"structural_tag\""); } } // The format field is required. if (obj.find("format") == obj.end()) { return ResultErr("Structural tag must have a format field"); } auto format = ParseFormat(obj["format"]); if (format.IsErr()) { return ResultErr(std::move(format).UnwrapErr()); } return ResultOk(std::move(format).Unwrap()); } Result StructuralTagParser::ParseFormat(const picojson::value& value) { RecursionGuard guard(&parse_format_recursion_depth_); if (!value.is()) { return ResultErr("Format must be an object"); } const auto& obj = value.get(); // If type is present, use it to determine the format. if (obj.find("type") != obj.end()) { if (!obj["type"].is()) { return ResultErr("Format's type must be a string"); } auto type = obj["type"].get(); if (type == "const_string") { return Result::Convert(ParseConstStringFormat(obj)); } else if (type == "json_schema") { return Result::Convert(ParseJSONSchemaFormat(obj)); } else if (type == "any_text") { return Result::Convert(ParseAnyTextFormat(obj)); } else if (type == "sequence") { return Result::Convert(ParseSequenceFormat(obj)); } else if (type == "or") { return Result::Convert(ParseOrFormat(obj)); } else if (type == "tag") { return Result::Convert(ParseTagFormat(obj)); } else if (type == "triggered_tags") { return Result::Convert(ParseTriggeredTagsFormat(obj)); } else if (type == "tags_with_separator") { return Result::Convert(ParseTagsWithSeparatorFormat(obj)); } else if (type == "optional") { return Result::Convert(ParseOptionalFormat(obj)); } else if (type == "plus") { return Result::Convert(ParsePlusFormat(obj)); } else if (type == "star") { return Result::Convert(ParseStarFormat(obj)); } else if (type == "repeat") { return Result::Convert(ParseRepeatFormat(obj)); } else if (type == "qwen_xml_parameter") { return Result::Convert(ParseJSONSchemaFormat(obj, "qwen_xml")); } else if (type == "grammar") { return Result::Convert(ParseGrammarFormat(obj)); } else if (type == "regex") { return Result::Convert(ParseRegexFormat(obj)); } else if (type == "token") { return Result::Convert(ParseTokenFormat(obj)); } else if (type == "exclude_token") { return Result::Convert(ParseExcludeTokenFormat(obj)); } else if (type == "any_tokens") { return Result::Convert(ParseAnyTokensFormat(obj)); } else if (type == "token_triggered_tags") { return Result::Convert(ParseTokenTriggeredTagsFormat(obj)); } else if (type == "dispatch") { return Result::Convert(ParseDispatchFormat(obj)); } else if (type == "token_dispatch") { return Result::Convert(ParseTokenDispatchFormat(obj)); } else { return ResultErr("Format type not recognized: " + type); } } // If type is not present, try every format type one by one. Tag is prioritized. auto tag_format = ParseTagFormat(obj); if (!tag_format.IsErr()) { return ResultOk(std::move(tag_format).Unwrap()); } auto const_string_format = ParseConstStringFormat(obj); if (!const_string_format.IsErr()) { return ResultOk(std::move(const_string_format).Unwrap()); } auto json_schema_format = ParseJSONSchemaFormat(obj); if (!json_schema_format.IsErr()) { return ResultOk(std::move(json_schema_format).Unwrap()); } auto any_text_format = ParseAnyTextFormat(obj); if (!any_text_format.IsErr()) { return ResultOk(std::move(any_text_format).Unwrap()); } auto sequence_format = ParseSequenceFormat(obj); if (!sequence_format.IsErr()) { return ResultOk(std::move(sequence_format).Unwrap()); } auto or_format = ParseOrFormat(obj); if (!or_format.IsErr()) { return ResultOk(std::move(or_format).Unwrap()); } auto triggered_tags_format = ParseTriggeredTagsFormat(obj); if (!triggered_tags_format.IsErr()) { return ResultOk(std::move(triggered_tags_format).Unwrap()); } auto tags_with_separator_format = ParseTagsWithSeparatorFormat(obj); if (!tags_with_separator_format.IsErr()) { return ResultOk(std::move(tags_with_separator_format).Unwrap()); } auto optional_format = ParseOptionalFormat(obj); if (!optional_format.IsErr()) { return ResultOk(std::move(optional_format).Unwrap()); } auto plus_format = ParsePlusFormat(obj); if (!plus_format.IsErr()) { return ResultOk(std::move(plus_format).Unwrap()); } auto star_format = ParseStarFormat(obj); if (!star_format.IsErr()) { return ResultOk(std::move(star_format).Unwrap()); } auto repeat_format = ParseRepeatFormat(obj); if (!repeat_format.IsErr()) { return ResultOk(std::move(repeat_format).Unwrap()); } auto tag_dispatch_format = ParseDispatchFormat(obj); if (!tag_dispatch_format.IsErr()) { return ResultOk(std::move(tag_dispatch_format).Unwrap()); } auto token_tag_dispatch_format = ParseTokenDispatchFormat(obj); if (!token_tag_dispatch_format.IsErr()) { return ResultOk(std::move(token_tag_dispatch_format).Unwrap()); } return ResultErr("Invalid format: " + value.serialize(false)); } Result StructuralTagParser::ParseConstStringFormat( const picojson::object& obj ) { // value is required. auto value_it = obj.find("value"); if (value_it == obj.end() || !value_it->second.is()) { return ResultErr("ConstString format must have a value field with a string"); } return ResultOk(value_it->second.get()); } Result StructuralTagParser::ParseJSONSchemaFormat( const picojson::object& obj, std::optional style_override ) { // json_schema is required. auto json_schema_it = obj.find("json_schema"); if (json_schema_it == obj.end() || !(json_schema_it->second.is() || json_schema_it->second.is())) { return ResultErr( "JSON schema format must have a json_schema field with a object or boolean value" ); } std::string style = "json"; if (style_override.has_value()) { style = *style_override; } else { auto it = obj.find("style"); if (it != obj.end() && it->second.is()) { style = it->second.get(); if (style != "json" && style != "qwen_xml" && style != "minimax_xml" && style != "deepseek_xml" && style != "glm_xml") { return ResultErr( "style must be \"json\", \"qwen_xml\", \"minimax_xml\", \"deepseek_xml\", or " "\"glm_xml\"" ); } } } bool any_order = false; auto any_order_it = obj.find("any_order"); if (any_order_it != obj.end()) { if (!any_order_it->second.is()) { return ResultErr("any_order must be a boolean"); } any_order = any_order_it->second.get(); } // here introduces a serialization/deserialization overhead; try to avoid it in the future. return ResultOk(json_schema_it->second.serialize(false), style, any_order); } Result StructuralTagParser::ParseAnyTextFormat(const picojson::object& obj ) { auto excluded_strs_it = obj.find("excludes"); if (excluded_strs_it == obj.end()) { if ((obj.find("type") == obj.end())) { return ResultErr("Any text format should not have any fields other than type"); } return ResultOk(std::vector{}); } if (!excluded_strs_it->second.is()) { return ResultErr("AnyText format's excluded_strs field must be an array"); } const auto& excluded_strs_array = excluded_strs_it->second.get(); std::vector excluded_strs; excluded_strs.reserve(excluded_strs_array.size()); for (const auto& excluded_str : excluded_strs_array) { if (!excluded_str.is()) { return ResultErr("AnyText format's excluded_strs array must contain strings"); } excluded_strs.push_back(excluded_str.get()); } return ResultOk(std::move(excluded_strs)); } Result StructuralTagParser::ParseGrammarFormat(const picojson::object& obj ) { // grammar is required. auto grammar_it = obj.find("grammar"); if (grammar_it == obj.end() || !grammar_it->second.is() || grammar_it->second.get().empty()) { return ResultErr("Grammar format must have a grammar field with a non-empty string"); } return ResultOk(grammar_it->second.get()); } Result StructuralTagParser::ParseRegexFormat(const picojson::object& obj) { // pattern is required. auto pattern_it = obj.find("pattern"); if (pattern_it == obj.end() || !pattern_it->second.is() || pattern_it->second.get().empty()) { return ResultErr("Regex format must have a pattern field with a non-empty string"); } return ResultOk(pattern_it->second.get()); } Result StructuralTagParser::ParseSequenceFormat( const picojson::object& obj ) { // elements is required. auto elements_it = obj.find("elements"); if (elements_it == obj.end() || !elements_it->second.is()) { return ResultErr("Sequence format must have an elements field with an array"); } const auto& elements_array = elements_it->second.get(); std::vector elements; elements.reserve(elements_array.size()); for (const auto& element : elements_array) { auto format = ParseFormat(element); if (format.IsErr()) { return ResultErr(std::move(format).UnwrapErr()); } elements.push_back(std::move(format).Unwrap()); } if (elements.size() == 0) { return ResultErr("Sequence format must have at least one element"); } return ResultOk(std::move(elements)); } Result StructuralTagParser::ParseOrFormat(const picojson::object& obj) { // elements is required. auto elements_it = obj.find("elements"); if (elements_it == obj.end() || !elements_it->second.is()) { return ResultErr("Or format must have an elements field with an array"); } const auto& elements_array = elements_it->second.get(); std::vector elements; elements.reserve(elements_array.size()); for (const auto& element : elements_array) { auto format = ParseFormat(element); if (format.IsErr()) { return ResultErr(std::move(format).UnwrapErr()); } elements.push_back(std::move(format).Unwrap()); } if (elements.size() == 0) { return ResultErr("Or format must have at least one element"); } return ResultOk(std::move(elements)); } Result StructuralTagParser::ParseTagFormat(const picojson::value& value) { if (!value.is()) { return ResultErr("Tag format must be an object"); } const auto& obj = value.get(); if (obj.find("type") != obj.end() && (!obj["type"].is() || obj["type"].get() != "tag")) { return ResultErr("Tag format's type must be a string \"tag\""); } return ParseTagFormat(obj); } Result StructuralTagParser::ParseTagFormat(const picojson::object& obj) { // begin is required: string or TokenFormat object auto begin_it = obj.find("begin"); if (begin_it == obj.end()) { return ResultErr("Tag format's begin field must be a string"); } std::variant begin; if (begin_it->second.is()) { begin = begin_it->second.get(); } else if (begin_it->second.is()) { auto tf = ParseTokenFormat(begin_it->second.get()); if (tf.IsErr()) { return ResultErr(std::move(tf).UnwrapErr()); } begin = std::move(tf).Unwrap(); } else { return ResultErr("Tag format's begin field must be a string"); } // content is required. auto content_it = obj.find("content"); if (content_it == obj.end()) { return ResultErr("Tag format must have a content field"); } auto content = ParseFormat(content_it->second); if (content.IsErr()) { return ResultErr(std::move(content).UnwrapErr()); } // end is required: string, array of strings, or TokenFormat object auto end_it = obj.find("end"); if (end_it == obj.end()) { return ResultErr("Tag format must have an end field"); } std::variant, TokenFormat> end; if (end_it->second.is()) { end = std::vector{end_it->second.get()}; } else if (end_it->second.is()) { const auto& end_array = end_it->second.get(); if (end_array.empty()) { return ResultErr("Tag format's end array cannot be empty"); } std::vector end_strings; for (const auto& item : end_array) { if (!item.is()) { return ResultErr("Tag format's end array must contain only strings"); } end_strings.push_back(item.get()); } end = std::move(end_strings); } else if (end_it->second.is()) { auto tf = ParseTokenFormat(end_it->second.get()); if (tf.IsErr()) { return ResultErr(std::move(tf).UnwrapErr()); } end = std::move(tf).Unwrap(); } else { return ResultErr("Tag format's end field must be a string or array of strings"); } return ResultOk( std::move(begin), std::make_shared(std::move(content).Unwrap()), std::move(end) ); } Result StructuralTagParser::ParseTriggeredTagsFormat( const picojson::object& obj ) { // triggers is required. auto triggers_it = obj.find("triggers"); if (triggers_it == obj.end() || !triggers_it->second.is()) { return ResultErr("Triggered tags format must have a triggers field with an array"); } const auto& triggers_array = triggers_it->second.get(); std::vector excluded_strs; std::vector triggers; triggers.reserve(triggers_array.size()); for (const auto& trigger : triggers_array) { if (!trigger.is() || trigger.get().empty()) { return ResultErr("Triggered tags format's triggers must be non-empty strings"); } triggers.push_back(trigger.get()); } if (triggers.size() == 0) { return ResultErr("Triggered tags format's triggers must be non-empty"); } // tags is required. auto tags_it = obj.find("tags"); if (tags_it == obj.end() || !tags_it->second.is()) { return ResultErr("Triggered tags format must have a tags field with an array"); } const auto& tags_array = tags_it->second.get(); std::vector tags; tags.reserve(tags_array.size()); for (const auto& tag : tags_array) { auto tag_format = ParseTagFormat(tag); if (tag_format.IsErr()) { return ResultErr(std::move(tag_format).UnwrapErr()); } tags.push_back(std::move(tag_format).Unwrap()); } if (tags.size() == 0) { return ResultErr("Triggered tags format's tags must be non-empty"); } // excludes is optional. auto excludes_it = obj.find("excludes"); if (excludes_it != obj.end()) { if (!excludes_it->second.is()) { return ResultErr("Triggered tags format should have a excludes field with an array" ); } const auto& excludes_array = excludes_it->second.get(); excluded_strs.reserve(excludes_array.size()); for (const auto& excluded_str : excludes_array) { if (!excluded_str.is() || excluded_str.get().empty()) { return ResultErr("Triggered tags format's excluded_strs must be non-empty strings" ); } excluded_strs.push_back(excluded_str.get()); } } // at_least_one is optional. bool at_least_one = false; auto at_least_one_it = obj.find("at_least_one"); if (at_least_one_it != obj.end()) { if (!at_least_one_it->second.is()) { return ResultErr("at_least_one must be a boolean"); } at_least_one = at_least_one_it->second.get(); } // stop_after_first is optional. bool stop_after_first = false; auto stop_after_first_it = obj.find("stop_after_first"); if (stop_after_first_it != obj.end()) { if (!stop_after_first_it->second.is()) { return ResultErr("stop_after_first must be a boolean"); } stop_after_first = stop_after_first_it->second.get(); } return ResultOk( std::move(triggers), std::move(tags), std::move(excluded_strs), at_least_one, stop_after_first ); } Result StructuralTagParser::ParseTagsWithSeparatorFormat( const picojson::object& obj ) { // tags is required. auto tags_it = obj.find("tags"); if (tags_it == obj.end() || !tags_it->second.is()) { return ResultErr("Tags with separator format must have a tags field with an array"); } const auto& tags_array = tags_it->second.get(); std::vector tags; tags.reserve(tags_array.size()); for (const auto& tag : tags_array) { auto tag_format = ParseTagFormat(tag); if (tag_format.IsErr()) { return ResultErr(std::move(tag_format).UnwrapErr()); } tags.push_back(std::move(tag_format).Unwrap()); } if (tags.size() == 0) { return ResultErr("Tags with separator format's tags must be non-empty"); } // separator is required (can be empty string). auto separator_it = obj.find("separator"); if (separator_it == obj.end() || !separator_it->second.is()) { return ResultErr("Tags with separator format's separator field must be a string"); } // at_least_one is optional. bool at_least_one = false; auto at_least_one_it = obj.find("at_least_one"); if (at_least_one_it != obj.end()) { if (!at_least_one_it->second.is()) { return ResultErr("at_least_one must be a boolean"); } at_least_one = at_least_one_it->second.get(); } // stop_after_first is optional. bool stop_after_first = false; auto stop_after_first_it = obj.find("stop_after_first"); if (stop_after_first_it != obj.end()) { if (!stop_after_first_it->second.is()) { return ResultErr("stop_after_first must be a boolean"); } stop_after_first = stop_after_first_it->second.get(); } return ResultOk( std::move(tags), separator_it->second.get(), at_least_one, stop_after_first ); } Result StructuralTagParser::ParseOptionalFormat( const picojson::object& obj ) { auto content_it = obj.find("content"); if (content_it == obj.end()) { return ResultErr("Optional format must have a content field"); } auto content = ParseFormat(content_it->second); if (content.IsErr()) { return ResultErr(std::move(content).UnwrapErr()); } return ResultOk(std::make_shared(std::move(content).Unwrap())); } Result StructuralTagParser::ParsePlusFormat(const picojson::object& obj) { auto content_it = obj.find("content"); if (content_it == obj.end()) { return ResultErr("Plus format must have a content field"); } auto content = ParseFormat(content_it->second); if (content.IsErr()) { return ResultErr(std::move(content).UnwrapErr()); } return ResultOk(std::make_shared(std::move(content).Unwrap())); } Result StructuralTagParser::ParseStarFormat(const picojson::object& obj) { auto content_it = obj.find("content"); if (content_it == obj.end()) { return ResultErr("Star format must have a content field"); } auto content = ParseFormat(content_it->second); if (content.IsErr()) { return ResultErr(std::move(content).UnwrapErr()); } return ResultOk(std::make_shared(std::move(content).Unwrap())); } Result StructuralTagParser::ParseTokenFormat(const picojson::object& obj) { auto token_it = obj.find("token"); if (token_it == obj.end()) { return ResultErr("TokenFormat must have a token field"); } if (token_it->second.is()) { double d = token_it->second.get(); if (d != static_cast(static_cast(d))) { return ResultErr("Token ID must be an integer"); } int32_t id = static_cast(d); if (id < 0) { return ResultErr("Token ID must be non-negative"); } return ResultOk(std::variant(id)); } else if (token_it->second.is()) { auto s = token_it->second.get(); if (s.empty()) { return ResultErr("Token string must be non-empty"); } return ResultOk(std::variant(std::move(s))); } return ResultErr("TokenFormat's token must be an integer or string"); } Result>, ISTError> ParseIntOrStringArray( const picojson::value& val, const std::string& field_name ) { std::vector> result; if (!val.is()) { return ResultErr(field_name + " must be an array"); } for (const auto& v : val.get()) { if (v.is()) { double d = v.get(); if (d != static_cast(static_cast(d))) { return ResultErr(field_name + " elements must be integers, not floats"); } int32_t id = static_cast(d); if (id < 0) { return ResultErr( field_name + " elements must be non-negative integers or strings" ); } result.push_back(id); } else if (v.is()) { auto s = v.get(); if (s.empty()) { return ResultErr(field_name + " string elements must be non-empty"); } result.push_back(std::move(s)); } else { return ResultErr(field_name + " elements must be integers or strings"); } } return ResultOk(std::move(result)); } Result StructuralTagParser::ParseExcludeTokenFormat( const picojson::object& obj ) { std::vector> exclude_tokens; auto it = obj.find("exclude_tokens"); if (it != obj.end()) { auto parsed = ParseIntOrStringArray(it->second, "exclude_tokens"); if (parsed.IsErr()) { return ResultErr(std::move(parsed).UnwrapErr()); } exclude_tokens = std::move(parsed).Unwrap(); } return ResultOk(std::move(exclude_tokens)); } Result StructuralTagParser::ParseAnyTokensFormat( const picojson::object& obj ) { std::vector> exclude_tokens; auto it = obj.find("exclude_tokens"); if (it != obj.end()) { auto parsed = ParseIntOrStringArray(it->second, "exclude_tokens"); if (parsed.IsErr()) { return ResultErr(std::move(parsed).UnwrapErr()); } exclude_tokens = std::move(parsed).Unwrap(); } return ResultOk(std::move(exclude_tokens)); } Result StructuralTagParser::ParseTokenTriggeredTagsFormat( const picojson::object& obj ) { // trigger_tokens is required auto triggers_it = obj.find("trigger_tokens"); if (triggers_it == obj.end()) { return ResultErr("TokenTriggeredTagsFormat must have a trigger_tokens field"); } auto triggers = ParseIntOrStringArray(triggers_it->second, "trigger_tokens"); if (triggers.IsErr()) { return ResultErr(std::move(triggers).UnwrapErr()); } auto trigger_tokens = std::move(triggers).Unwrap(); if (trigger_tokens.empty()) { return ResultErr("trigger_tokens must be non-empty"); } // tags is required auto tags_it = obj.find("tags"); if (tags_it == obj.end() || !tags_it->second.is()) { return ResultErr("TokenTriggeredTagsFormat must have a tags field with an array"); } std::vector tags; for (const auto& tag : tags_it->second.get()) { auto tag_format = ParseTagFormat(tag); if (tag_format.IsErr()) { return ResultErr(std::move(tag_format).UnwrapErr()); } tags.push_back(std::move(tag_format).Unwrap()); } if (tags.empty()) { return ResultErr("TokenTriggeredTagsFormat tags must be non-empty"); } // exclude_tokens is optional std::vector> exclude_tokens; auto excludes_it = obj.find("exclude_tokens"); if (excludes_it != obj.end()) { auto parsed = ParseIntOrStringArray(excludes_it->second, "exclude_tokens"); if (parsed.IsErr()) { return ResultErr(std::move(parsed).UnwrapErr()); } exclude_tokens = std::move(parsed).Unwrap(); } bool at_least_one = false; auto alo_it = obj.find("at_least_one"); if (alo_it != obj.end()) { if (!alo_it->second.is()) { return ResultErr("at_least_one must be a boolean"); } at_least_one = alo_it->second.get(); } bool stop_after_first = false; auto saf_it = obj.find("stop_after_first"); if (saf_it != obj.end()) { if (!saf_it->second.is()) { return ResultErr("stop_after_first must be a boolean"); } stop_after_first = saf_it->second.get(); } return ResultOk( std::move(trigger_tokens), std::move(tags), std::move(exclude_tokens), at_least_one, stop_after_first ); } Result StructuralTagParser::ParseDispatchFormat( const picojson::object& obj ) { auto rules_it = obj.find("rules"); if (rules_it == obj.end() || !rules_it->second.is()) { return ResultErr("TagDispatch format must have a rules field with an array"); } const auto& rules_array = rules_it->second.get(); if (rules_array.empty()) { return ResultErr("TagDispatch format rules must be non-empty"); } std::vector>> rules; rules.reserve(rules_array.size()); for (const auto& item : rules_array) { if (!item.is()) { return ResultErr("TagDispatch pair must be a 2-element array"); } const auto& pair_arr = item.get(); if (pair_arr.size() != 2) { return ResultErr("TagDispatch pair must be a 2-element array"); } if (!pair_arr[0].is()) { return ResultErr("TagDispatch pair first element must be a string"); } std::string trigger = pair_arr[0].get(); auto content = ParseFormat(pair_arr[1]); if (content.IsErr()) { return ResultErr(std::move(content).UnwrapErr()); } rules.push_back({std::move(trigger), std::make_shared(std::move(content).Unwrap())}); } bool loop = true; auto loop_it = obj.find("loop"); if (loop_it != obj.end()) { if (!loop_it->second.is()) { return ResultErr("loop must be a boolean"); } loop = loop_it->second.get(); } std::vector excludes; auto excludes_it = obj.find("excludes"); if (excludes_it != obj.end()) { if (!excludes_it->second.is()) { return ResultErr("excludes must be an array"); } for (const auto& e : excludes_it->second.get()) { if (!e.is() || e.get().empty()) { return ResultErr("excludes must contain non-empty strings"); } excludes.push_back(e.get()); } } return ResultOk(std::move(rules), loop, std::move(excludes)); } Result StructuralTagParser::ParseTokenDispatchFormat( const picojson::object& obj ) { auto rules_it = obj.find("rules"); if (rules_it == obj.end() || !rules_it->second.is()) { return ResultErr("TokenTagDispatch format must have a rules field with an array"); } const auto& rules_array = rules_it->second.get(); if (rules_array.empty()) { return ResultErr("TokenTagDispatch format rules must be non-empty"); } std::vector, std::shared_ptr>> rules; rules.reserve(rules_array.size()); for (const auto& item : rules_array) { if (!item.is()) { return ResultErr("TokenTagDispatch pair must be a 2-element array"); } const auto& pair_arr = item.get(); if (pair_arr.size() != 2) { return ResultErr("TokenTagDispatch pair must be a 2-element array"); } std::variant trigger; if (pair_arr[0].is()) { double d = pair_arr[0].get(); if (d != static_cast(static_cast(d))) { return ResultErr("Token ID must be an integer"); } trigger = static_cast(d); } else if (pair_arr[0].is()) { trigger = pair_arr[0].get(); } else { return ResultErr("TokenTagDispatch pair first element must be an integer or string" ); } auto content = ParseFormat(pair_arr[1]); if (content.IsErr()) { return ResultErr(std::move(content).UnwrapErr()); } rules.push_back({std::move(trigger), std::make_shared(std::move(content).Unwrap())}); } bool loop = true; auto loop_it = obj.find("loop"); if (loop_it != obj.end()) { if (!loop_it->second.is()) { return ResultErr("loop must be a boolean"); } loop = loop_it->second.get(); } std::vector> exclude_tokens; auto excludes_it = obj.find("exclude_tokens"); if (excludes_it != obj.end()) { auto parsed = ParseIntOrStringArray(excludes_it->second, "exclude_tokens"); if (parsed.IsErr()) { return ResultErr(std::move(parsed).UnwrapErr()); } exclude_tokens = std::move(parsed).Unwrap(); } return ResultOk(std::move(rules), loop, std::move(exclude_tokens)); } /************** StructuralTagTokenResolver **************/ class StructuralTagTokenResolver { public: static std::optional Resolve( StructuralTag* structural_tag, const std::optional& tokenizer_info ); private: explicit StructuralTagTokenResolver(const std::optional& tokenizer_info) : tokenizer_info_(tokenizer_info) {} std::optional ResolveFormat(Format* format); std::optional ResolveTagFormat(TagFormat* tag); std::optional ResolveTokenFormat(TokenFormat* tf); std::optional ResolveIntOrStringVec( const std::vector>& input, std::vector* output ); const std::optional& tokenizer_info_; }; std::optional StructuralTagTokenResolver::Resolve( StructuralTag* structural_tag, const std::optional& tokenizer_info ) { return StructuralTagTokenResolver(tokenizer_info).ResolveFormat(&structural_tag->format); } std::optional StructuralTagTokenResolver::ResolveTokenFormat(TokenFormat* tf) { if (tf->resolved_token_id_ >= 0) return std::nullopt; if (!std::holds_alternative(tf->token)) return std::nullopt; if (!tokenizer_info_) { return ISTError("Token string resolution requires tokenizer_info"); } const auto& token_str = std::get(tf->token); const auto& vocab = tokenizer_info_->GetDecodedVocab(); for (int32_t i = 0; i < static_cast(vocab.size()); ++i) { if (vocab[i] == token_str) { tf->resolved_token_id_ = i; return std::nullopt; } } return ISTError("Token string \"" + token_str + "\" not found in vocabulary"); } std::optional StructuralTagTokenResolver::ResolveIntOrStringVec( const std::vector>& input, std::vector* output ) { output->clear(); output->reserve(input.size()); for (const auto& item : input) { if (std::holds_alternative(item)) { output->push_back(std::get(item)); } else { if (!tokenizer_info_) { return ISTError("Token string resolution requires tokenizer_info"); } const auto& s = std::get(item); const auto& vocab = tokenizer_info_->GetDecodedVocab(); bool found = false; for (int32_t i = 0; i < static_cast(vocab.size()); ++i) { if (vocab[i] == s) { output->push_back(i); found = true; break; } } if (!found) { return ISTError("Token string \"" + s + "\" not found in vocabulary"); } } } return std::nullopt; } std::optional StructuralTagTokenResolver::ResolveTagFormat(TagFormat* tag) { if (std::holds_alternative(tag->begin)) { auto err = ResolveTokenFormat(&std::get(tag->begin)); if (err) return err; } if (std::holds_alternative(tag->end)) { auto err = ResolveTokenFormat(&std::get(tag->end)); if (err) return err; } return ResolveFormat(tag->content.get()); } std::optional StructuralTagTokenResolver::ResolveFormat(Format* format) { return std::visit( [&](auto&& arg) -> std::optional { using T = std::decay_t; if constexpr (std::is_same_v) { return ResolveTokenFormat(&arg); } else if constexpr (std::is_same_v) { return ResolveIntOrStringVec(arg.exclude_tokens, &arg.resolved_token_ids_); } else if constexpr (std::is_same_v) { return ResolveIntOrStringVec(arg.exclude_tokens, &arg.resolved_exclude_token_ids_); } else if constexpr (std::is_same_v) { auto err = ResolveIntOrStringVec(arg.trigger_tokens, &arg.resolved_trigger_token_ids_); if (err) return err; err = ResolveIntOrStringVec(arg.exclude_tokens, &arg.resolved_exclude_token_ids_); if (err) return err; for (auto& tag : arg.tags) { err = ResolveTagFormat(&tag); if (err) return err; } return std::nullopt; } else if constexpr (std::is_same_v) { std::vector> trigger_tokens; trigger_tokens.reserve(arg.rules.size()); for (const auto& p : arg.rules) { trigger_tokens.push_back(p.first); } auto err = ResolveIntOrStringVec(trigger_tokens, &arg.resolved_trigger_token_ids_); if (err) return err; err = ResolveIntOrStringVec(arg.exclude_tokens, &arg.resolved_exclude_token_ids_); if (err) return err; for (auto& p : arg.rules) { if (p.second) { auto e = ResolveFormat(p.second.get()); if (e) return e; } } return std::nullopt; } else if constexpr (std::is_same_v) { for (auto& p : arg.rules) { if (p.second) { auto err = ResolveFormat(p.second.get()); if (err) return err; } } return std::nullopt; } else if constexpr (std::is_same_v) { return ResolveTagFormat(&arg); } else if constexpr (std::is_same_v) { for (auto& elem : arg.elements) { auto err = ResolveFormat(&elem); if (err) return err; } return std::nullopt; } else if constexpr (std::is_same_v) { for (auto& elem : arg.elements) { auto err = ResolveFormat(&elem); if (err) return err; } return std::nullopt; } else if constexpr (std::is_same_v) { for (auto& tag : arg.tags) { auto err = ResolveTagFormat(&tag); if (err) return err; } return std::nullopt; } else if constexpr (std::is_same_v) { for (auto& tag : arg.tags) { auto err = ResolveTagFormat(&tag); if (err) return err; } return std::nullopt; } else if constexpr (std::is_same_v || std::is_same_v || std::is_same_v) { return ResolveFormat(arg.content.get()); } else { return std::nullopt; } }, *format ); } Result StructuralTagParser::ParseRepeatFormat(const picojson::object& obj) { auto min_it = obj.find("min"); if (min_it == obj.end() || !min_it->second.is()) { return ResultErr("Repeat format must have a min field (number)"); } auto max_it = obj.find("max"); if (max_it == obj.end() || !max_it->second.is()) { return ResultErr("Repeat format must have a max field (number)"); } int64_t min = min_it->second.get(); int64_t max = max_it->second.get(); int32_t max_value_int32 = std::numeric_limits::max(); if (max >= 0 && min > max) { return ResultErr("Repeat min must be <= max"); } if (min < 0) { return ResultErr("Repeat min must be >= 0"); } if (max < -1) { return ResultErr("Repeat max must be -1 (unbounded) or >= 0"); } if (max > static_cast(max_value_int32)) { XGRAMMAR_LOG(WARNING) << "Repeat max is too large, will be set as not limited"; max = -1; // -1 means unlimited } if (min > static_cast(max_value_int32)) { return ResultErr( "Repeat min is too large, must be <= " + std::to_string(max_value_int32) ); } auto content_it = obj.find("content"); if (content_it == obj.end()) { return ResultErr("Repeat format must have a content field"); } auto content = ParseFormat(content_it->second); if (content.IsErr()) { return ResultErr(std::move(content).UnwrapErr()); } return ResultOk( static_cast(min), static_cast(max), std::make_shared(std::move(content).Unwrap()) ); } /************** StructuralTag Analyzer **************/ /*! * \brief Analyze a StructuralTag and extract useful information for conversion to Grammar. */ class StructuralTagAnalyzer { public: static std::optional Analyze(StructuralTag* structural_tag); private: /*! \brief A variant that can hold the pointer of any Format types. */ using FormatPtrVariant = std::variant< ConstStringFormat*, JSONSchemaFormat*, AnyTextFormat*, GrammarFormat*, RegexFormat*, SequenceFormat*, OrFormat*, TagFormat*, TriggeredTagsFormat*, TagsWithSeparatorFormat*, OptionalFormat*, PlusFormat*, StarFormat*, RepeatFormat*, TokenFormat*, ExcludeTokenFormat*, AnyTokensFormat*, TokenTriggeredTagsFormat*, DispatchFormat*, TokenDispatchFormat*>; // Call this if we have a pointer to a Format. std::optional Visit(Format* format); // Call this if we have a pointer to a variant of Format. std::optional Visit(FormatPtrVariant format); // The following is dispatched from Visit. Don't call them directly because they don't handle // stack logics. std::optional VisitSub(ConstStringFormat* format); std::optional VisitSub(JSONSchemaFormat* format); std::optional VisitSub(AnyTextFormat* format); std::optional VisitSub(GrammarFormat* format); std::optional VisitSub(RegexFormat* format); std::optional VisitSub(SequenceFormat* format); std::optional VisitSub(OrFormat* format); std::optional VisitSub(TagFormat* format); std::optional VisitSub(TriggeredTagsFormat* format); std::optional VisitSub(TagsWithSeparatorFormat* format); std::optional VisitSub(OptionalFormat* format); std::optional VisitSub(PlusFormat* format); std::optional VisitSub(StarFormat* format); std::optional VisitSub(TokenFormat* format); std::optional VisitSub(ExcludeTokenFormat* format); std::optional VisitSub(AnyTokensFormat* format); std::optional VisitSub(TokenTriggeredTagsFormat* format); std::optional VisitSub(RepeatFormat* format); std::optional VisitSub(DispatchFormat* format); std::optional VisitSub(TokenDispatchFormat* format); std::vector DetectEndStrings(); std::vector DetectEndTokenIds(); bool IsUnlimited(const Format& format); bool IsExcluded(const Format& format); int visit_format_recursion_depth_ = 0; std::vector stack_; }; std::optional StructuralTagAnalyzer::Analyze(StructuralTag* structural_tag) { return StructuralTagAnalyzer().Visit(&structural_tag->format); } std::vector StructuralTagAnalyzer::DetectEndStrings() { for (int i = static_cast(stack_.size()) - 1; i >= 0; --i) { auto& format = stack_[i]; if (std::holds_alternative(format)) { auto* tag = std::get(format); if (std::holds_alternative>(tag->end)) { return std::get>(tag->end); } return {}; // TokenFormat end — propagated via DetectEndTokenIds } } return {}; } std::vector StructuralTagAnalyzer::DetectEndTokenIds() { for (int i = static_cast(stack_.size()) - 1; i >= 0; --i) { auto& format = stack_[i]; if (std::holds_alternative(format)) { auto* tag = std::get(format); if (std::holds_alternative(tag->end)) { auto& tf = std::get(tag->end); return {tf.resolved_token_id_}; } return {}; } } return {}; } bool StructuralTagAnalyzer::IsUnlimited(const Format& format) { return std::visit( [&](auto&& arg) -> bool { using T = std::decay_t; if constexpr (std::is_same_v) { return true; } else if constexpr (std::is_same_v) { return true; } else if constexpr (std::is_same_v) { return true; } else if constexpr (std::is_same_v) { return true; } else if constexpr (std::is_same_v) { return true; } else if constexpr (std::is_same_v) { return true; } else if constexpr (std::is_same_v) { return true; } else if constexpr (std::is_same_v) { return arg.is_unlimited_; } else if constexpr (std::is_same_v) { return arg.is_unlimited_; } else if constexpr (std::is_same_v) { return IsUnlimited(*arg.content); } else if constexpr (std::is_same_v || std::is_same_v) { return true; } else if constexpr (std::is_same_v) { return arg.max == -1 || (arg.max != 0 && IsUnlimited(*arg.content)); } else { return false; } }, format ); } bool StructuralTagAnalyzer::IsExcluded(const Format& format) { return std::visit( [&](auto&& arg) -> bool { using T = std::decay_t; if constexpr (std::is_same_v) { return !arg.excludes.empty(); } else if constexpr (std::is_same_v) { return !arg.excludes.empty(); } else if constexpr (std::is_same_v) { return !arg.exclude_tokens.empty(); } else if constexpr (std::is_same_v) { return !arg.excludes.empty(); } else if constexpr (std::is_same_v) { return !arg.exclude_tokens.empty(); } else if constexpr (std::is_same_v) { return !arg.exclude_tokens.empty(); } else { return false; } }, format ); } std::optional StructuralTagAnalyzer::Visit(Format* format) { FormatPtrVariant format_ptr_variant = std::visit([&](auto&& arg) -> FormatPtrVariant { return &arg; }, *format); return Visit(format_ptr_variant); } std::optional StructuralTagAnalyzer::Visit(FormatPtrVariant format) { RecursionGuard guard(&visit_format_recursion_depth_); // Push format to stack stack_.push_back(format); // Dispatch to the corresponding visit function auto result = std::visit([&](auto&& arg) -> std::optional { return VisitSub(arg); }, format); // Pop format from stack stack_.pop_back(); return result; } std::optional StructuralTagAnalyzer::VisitSub(ConstStringFormat* format) { return std::nullopt; } std::optional StructuralTagAnalyzer::VisitSub(JSONSchemaFormat* format) { return std::nullopt; } std::optional StructuralTagAnalyzer::VisitSub(AnyTextFormat* format) { format->detected_end_strs_ = DetectEndStrings(); return std::nullopt; } std::optional StructuralTagAnalyzer::VisitSub(GrammarFormat* format) { return std::nullopt; } std::optional StructuralTagAnalyzer::VisitSub(RegexFormat* format) { return std::nullopt; } std::optional StructuralTagAnalyzer::VisitSub(SequenceFormat* format) { bool is_any_unlimited = false; for (auto& element : format->elements) { auto err = Visit(&element); if (err.has_value()) { return err; } is_any_unlimited |= IsUnlimited(element) && !IsExcluded(element); } format->is_unlimited_ = is_any_unlimited; return std::nullopt; } std::optional StructuralTagAnalyzer::VisitSub(OrFormat* format) { bool is_any_unlimited = false; for (auto& element : format->elements) { auto err = Visit(&element); if (err.has_value()) { return err; } is_any_unlimited |= IsUnlimited(element) && !IsExcluded(element); } format->is_unlimited_ = is_any_unlimited; return std::nullopt; } std::optional StructuralTagAnalyzer::VisitSub(TagFormat* format) { auto err = Visit(format->content.get()); if (err.has_value()) { return err; } auto is_content_unlimited = IsUnlimited(*(format->content)); if (is_content_unlimited) { if (std::holds_alternative>(format->end)) { const auto& ends = std::get>(format->end); bool has_non_empty_end = false; for (const auto& end_str : ends) { if (!end_str.empty()) { has_non_empty_end = true; break; } } if (!has_non_empty_end && !IsExcluded(*format->content)) { return ISTError("When the content is unlimited, at least one end string must be non-empty"); } } // TokenFormat end is always non-empty → no error needed } return std::nullopt; } std::optional StructuralTagAnalyzer::VisitSub(TriggeredTagsFormat* format) { for (auto& tag : format->tags) { auto err = Visit(&tag); if (err.has_value()) { return err; } } format->detected_end_strs_ = DetectEndStrings(); return std::nullopt; } std::optional StructuralTagAnalyzer::VisitSub(TagsWithSeparatorFormat* format) { for (auto& tag : format->tags) { auto err = Visit(&tag); if (err.has_value()) { return err; } } return std::nullopt; } std::optional StructuralTagAnalyzer::VisitSub(OptionalFormat* format) { return Visit(format->content.get()); } std::optional StructuralTagAnalyzer::VisitSub(PlusFormat* format) { return Visit(format->content.get()); } std::optional StructuralTagAnalyzer::VisitSub(StarFormat* format) { return Visit(format->content.get()); } std::optional StructuralTagAnalyzer::VisitSub(TokenFormat* format) { return std::nullopt; } std::optional StructuralTagAnalyzer::VisitSub(ExcludeTokenFormat* format) { format->detected_end_token_ids_ = DetectEndTokenIds(); return std::nullopt; } std::optional StructuralTagAnalyzer::VisitSub(AnyTokensFormat* format) { format->detected_end_token_ids_ = DetectEndTokenIds(); return std::nullopt; } std::optional StructuralTagAnalyzer::VisitSub(TokenTriggeredTagsFormat* format) { for (auto& tag : format->tags) { auto err = Visit(&tag); if (err.has_value()) { return err; } } format->detected_end_token_ids_ = DetectEndTokenIds(); return std::nullopt; } std::optional StructuralTagAnalyzer::VisitSub(DispatchFormat* format) { for (auto& pair : format->rules) { if (pair.second) { auto err = Visit(pair.second.get()); if (err.has_value()) return err; } } return std::nullopt; } std::optional StructuralTagAnalyzer::VisitSub(TokenDispatchFormat* format) { for (auto& pair : format->rules) { if (pair.second) { auto err = Visit(pair.second.get()); if (err.has_value()) return err; } } return std::nullopt; } std::optional StructuralTagAnalyzer::VisitSub(RepeatFormat* format) { return Visit(format->content.get()); } /************** StructuralTag to Grammar Converter **************/ class StructuralTagGrammarConverter { public: static Result Convert(const StructuralTag& structural_tag); private: /*! * \brief Visit a Format and return the rule id of the added rule. * \param format The Format to visit. * \return The rule id of the added rule. If the visit fails, the error is returned. * \note This method uses serialization to deduplicate identical formats. */ Result Visit(const Format& format); Result VisitSub(const ConstStringFormat& format); Result VisitSub(const JSONSchemaFormat& format); Result VisitSub(const AnyTextFormat& format); Result VisitSub(const GrammarFormat& format); Result VisitSub(const RegexFormat& format); Result VisitSub(const SequenceFormat& format); Result VisitSub(const OrFormat& format); Result VisitSub(const TagFormat& format); Result VisitSub(const TriggeredTagsFormat& format); Result VisitSub(const TagsWithSeparatorFormat& format); Result VisitSub(const OptionalFormat& format); Result VisitSub(const PlusFormat& format); Result VisitSub(const StarFormat& format); Result VisitSub(const TokenFormat& format); Result VisitSub(const ExcludeTokenFormat& format); Result VisitSub(const AnyTokensFormat& format); Result VisitSub(const TokenTriggeredTagsFormat& format); Result VisitSub(const RepeatFormat& format); Result VisitSub(const DispatchFormat& format); Result VisitSub(const TokenDispatchFormat& format); Grammar AddRootRuleAndGetGrammar(int ref_rule_id); bool IsPrefix(const std::string& prefix, const std::string& full_str); int BuildBeginExpr(const TagFormat& tag); int BuildEndExpr(const TagFormat& tag); GrammarBuilder grammar_builder_; /*! * \brief Cache from format serialization to rule id. * This enables deduplication of identical formats to reduce grammar size. */ std::unordered_map serialization_to_rule_id_; }; bool StructuralTagGrammarConverter::IsPrefix( const std::string& prefix, const std::string& full_str ) { return prefix.size() <= full_str.size() && std::string_view(full_str).substr(0, prefix.size()) == prefix; } Result StructuralTagGrammarConverter::Convert(const StructuralTag& structural_tag ) { StructuralTagGrammarConverter converter; auto result = converter.Visit(structural_tag.format); if (result.IsErr()) { return ResultErr(std::move(result).UnwrapErr()); } // Add a root rule auto root_rule_id = std::move(result).Unwrap(); return ResultOk(converter.AddRootRuleAndGetGrammar(root_rule_id)); } Grammar StructuralTagGrammarConverter::AddRootRuleAndGetGrammar(int ref_rule_id) { auto expr = grammar_builder_.AddRuleRef(ref_rule_id); auto sequence_expr = grammar_builder_.AddSequence({expr}); auto choices_expr = grammar_builder_.AddChoices({sequence_expr}); auto root_rule_id = grammar_builder_.AddRuleWithHint("root", choices_expr); return grammar_builder_.Get(root_rule_id); } Result StructuralTagGrammarConverter::Visit(const Format& format) { std::string fingerprint = FormatToJSONValue(format).serialize(); // Check if we've already processed an identical format auto it = serialization_to_rule_id_.find(fingerprint); if (it != serialization_to_rule_id_.end()) { return ResultOk(it->second); } // Process the format and cache the result auto result = std::visit([&](auto&& arg) -> Result { return VisitSub(arg); }, format); if (result.IsOk()) { int rule_id = std::move(result).Unwrap(); serialization_to_rule_id_[fingerprint] = rule_id; return ResultOk(rule_id); } return result; } Result StructuralTagGrammarConverter::VisitSub(const ConstStringFormat& format) { auto expr = format.value.empty() ? grammar_builder_.AddEmptyStr() : grammar_builder_.AddByteString(format.value); auto sequence_expr = grammar_builder_.AddSequence({expr}); auto choices_expr = grammar_builder_.AddChoices({sequence_expr}); return ResultOk(grammar_builder_.AddRuleWithHint("const_string", choices_expr)); } Result StructuralTagGrammarConverter::VisitSub(const JSONSchemaFormat& format) { const static std::unordered_map> style_to_grammar_converter = { {"json", [](const std::string& json_schema, bool any_order) -> std::string { return JSONSchemaToEBNF( json_schema, /*any_whitespace=*/true, /*indent=*/std::nullopt, /*separators=*/std::nullopt, /*strict_mode=*/true, /*max_whitespace_cnt=*/std::nullopt, /*json_format=*/JSONFormat::kJSON, any_order ); }}, {"qwen_xml", [](const std::string& json_schema, bool any_order) -> std::string { return QwenXMLToolCallingToEBNF(json_schema, any_order); }}, {"minimax_xml", [](const std::string& json_schema, bool any_order) -> std::string { return MiniMaxXMLToolCallingToEBNF(json_schema, any_order); }}, {"deepseek_xml", [](const std::string& json_schema, bool any_order) -> std::string { return DeepSeekXMLToolCallingToEBNF(json_schema, any_order); }}, {"glm_xml", [](const std::string& json_schema, bool any_order) -> std::string { return GlmXMLToolCallingToEBNF(json_schema, any_order); }}, }; auto converter = style_to_grammar_converter.find(format.style); if (converter == style_to_grammar_converter.end()) { return ResultErr("Unsupported parsing type: " + format.style); } std::string ebnf = converter->second(format.json_schema, format.any_order); auto sub_grammar = Grammar::FromEBNF(ebnf); auto added_root_rule_id = SubGrammarAdder().Apply(&grammar_builder_, sub_grammar); return ResultOk(added_root_rule_id); } Result StructuralTagGrammarConverter::VisitSub(const GrammarFormat& format) { auto sub_grammar = Grammar::FromEBNF(format.grammar); auto added_root_rule_id = SubGrammarAdder().Apply(&grammar_builder_, sub_grammar); return ResultOk(added_root_rule_id); } Result StructuralTagGrammarConverter::VisitSub(const RegexFormat& format) { auto sub_grammar = Grammar::FromRegex(format.pattern); auto added_root_rule_id = SubGrammarAdder().Apply(&grammar_builder_, sub_grammar); return ResultOk(added_root_rule_id); } Result StructuralTagGrammarConverter::VisitSub(const AnyTextFormat& format) { std::vector all_excludes = format.excludes; for (const auto& s : format.detected_end_strs_) { if (!s.empty()) { all_excludes.push_back(s); } } if (!all_excludes.empty()) { auto tag_dispatch_expr = grammar_builder_.AddTagDispatch(Grammar::Impl::TagDispatch{{}, false, all_excludes}); return ResultOk(grammar_builder_.AddRuleWithHint("any_text", tag_dispatch_expr)); } else { auto any_text_expr = grammar_builder_.AddCharacterClassStar({{0, 0x10FFFF}}, false); auto sequence_expr = grammar_builder_.AddSequence({any_text_expr}); auto choices_expr = grammar_builder_.AddChoices({sequence_expr}); return ResultOk(grammar_builder_.AddRuleWithHint("any_text", choices_expr)); } } Result StructuralTagGrammarConverter::VisitSub(const SequenceFormat& format) { std::vector rule_ref_ids; rule_ref_ids.reserve(format.elements.size()); for (const auto& element : format.elements) { auto result = Visit(element); if (result.IsErr()) { return result; } int sub_rule_id = std::move(result).Unwrap(); rule_ref_ids.push_back(grammar_builder_.AddRuleRef(sub_rule_id)); } auto expr = grammar_builder_.AddChoices({grammar_builder_.AddSequence(rule_ref_ids)}); return ResultOk(grammar_builder_.AddRuleWithHint("sequence", expr)); } Result StructuralTagGrammarConverter::VisitSub(const OrFormat& format) { std::vector sequence_ids; sequence_ids.reserve(format.elements.size()); for (const auto& element : format.elements) { auto result = Visit(element); if (result.IsErr()) { return result; } int sub_rule_id = std::move(result).Unwrap(); auto rule_ref_expr = grammar_builder_.AddRuleRef(sub_rule_id); sequence_ids.push_back(grammar_builder_.AddSequence({rule_ref_expr})); } auto expr = grammar_builder_.AddChoices(sequence_ids); return ResultOk(grammar_builder_.AddRuleWithHint("or", expr)); } int StructuralTagGrammarConverter::BuildBeginExpr(const TagFormat& tag) { if (std::holds_alternative(tag.begin)) { return grammar_builder_.AddByteString(std::get(tag.begin)); } return grammar_builder_.AddTokenSet({std::get(tag.begin).resolved_token_id_}); } int StructuralTagGrammarConverter::BuildEndExpr(const TagFormat& tag) { if (std::holds_alternative(tag.end)) { return grammar_builder_.AddTokenSet({std::get(tag.end).resolved_token_id_}); } const auto& ends = std::get>(tag.end); if (ends.size() == 1) { return ends[0].empty() ? grammar_builder_.AddEmptyStr() : grammar_builder_.AddByteString(ends[0]); } std::vector end_seq_ids; for (const auto& s : ends) { auto e = s.empty() ? grammar_builder_.AddEmptyStr() : grammar_builder_.AddByteString(s); end_seq_ids.push_back(grammar_builder_.AddSequence({e})); } auto choice = grammar_builder_.AddChoices(end_seq_ids); auto rule = grammar_builder_.AddRuleWithHint("tag_end", choice); return grammar_builder_.AddRuleRef(rule); } Result StructuralTagGrammarConverter::VisitSub(const TagFormat& format) { auto result = Visit(*format.content); if (result.IsErr()) { return result; } auto sub_rule_id = std::move(result).Unwrap(); auto begin_expr = BuildBeginExpr(format); auto rule_ref_expr = grammar_builder_.AddRuleRef(sub_rule_id); auto end_expr = BuildEndExpr(format); auto sequence_expr_id = grammar_builder_.AddSequence({begin_expr, rule_ref_expr, end_expr}); auto choices_expr = grammar_builder_.AddChoices({sequence_expr_id}); return ResultOk(grammar_builder_.AddRuleWithHint("tag", choices_expr)); } Result StructuralTagGrammarConverter::VisitSub(const TriggeredTagsFormat& format) { // Step 1. Visit all tags and add to grammar std::vector> trigger_to_tag_ids(format.triggers.size()); std::vector tag_content_rule_ids; tag_content_rule_ids.reserve(format.tags.size()); for (int it_tag = 0; it_tag < static_cast(format.tags.size()); ++it_tag) { const auto& tag = format.tags[it_tag]; if (!std::holds_alternative(tag.begin)) { return ResultErr( "Tags in triggered_tags must have a string begin, not a token format" ); } const auto& tag_begin = std::get(tag.begin); int matched_trigger_id = -1; for (int it_trigger = 0; it_trigger < static_cast(format.triggers.size()); ++it_trigger) { const auto& trigger = format.triggers[it_trigger]; if (IsPrefix(trigger, tag_begin)) { if (matched_trigger_id != -1) { return ResultErr("One tag matches multiple triggers in a triggered tags format" ); } matched_trigger_id = it_trigger; } } if (matched_trigger_id == -1) { return ResultErr("One tag does not match any trigger in a triggered tags format"); } trigger_to_tag_ids[matched_trigger_id].push_back(it_tag); auto result = Visit(*tag.content); if (result.IsErr()) { return result; } tag_content_rule_ids.push_back(std::move(result).Unwrap()); } // Step 2. Special Case: at_least_one && stop_after_first. if (format.at_least_one && format.stop_after_first) { std::vector choice_elements; for (int it_tag = 0; it_tag < static_cast(format.tags.size()); ++it_tag) { const auto& tag = format.tags[it_tag]; auto begin_expr_id = BuildBeginExpr(tag); auto rule_ref_expr_id = grammar_builder_.AddRuleRef(tag_content_rule_ids[it_tag]); auto end_expr_id = BuildEndExpr(tag); choice_elements.push_back( grammar_builder_.AddSequence({begin_expr_id, rule_ref_expr_id, end_expr_id}) ); } auto choice_expr_id = grammar_builder_.AddChoices(choice_elements); return ResultOk(grammar_builder_.AddRuleWithHint("triggered_tags", choice_expr_id)); } // Step 3. Normal Case. // Step 3.1 Get tag_rule_pairs. std::vector> tag_rule_pairs; for (int it_trigger = 0; it_trigger < static_cast(format.triggers.size()); ++it_trigger) { const auto& trigger = format.triggers[it_trigger]; std::vector choice_elements; for (const auto& tag_id : trigger_to_tag_ids[it_trigger]) { const auto& tag = format.tags[tag_id]; const auto& tag_begin = std::get(tag.begin); int begin_expr_id = grammar_builder_.AddByteString(tag_begin.substr(trigger.size())); int rule_ref_expr_id = grammar_builder_.AddRuleRef(tag_content_rule_ids[tag_id]); int end_expr_id = BuildEndExpr(tag); choice_elements.push_back( grammar_builder_.AddSequence({begin_expr_id, rule_ref_expr_id, end_expr_id}) ); } auto choice_expr_id = grammar_builder_.AddChoices(choice_elements); auto sub_rule_id = grammar_builder_.AddRuleWithHint("triggered_tags_group", choice_expr_id); tag_rule_pairs.push_back(std::make_pair(trigger, sub_rule_id)); } // Step 3.2 Add TagDispatch. int32_t rule_expr_id; bool loop_after_dispatch = !format.stop_after_first; std::vector all_excludes = format.excludes; for (const auto& s : format.detected_end_strs_) { if (!s.empty()) { all_excludes.push_back(s); } } rule_expr_id = grammar_builder_.AddTagDispatch( Grammar::Impl::TagDispatch{tag_rule_pairs, loop_after_dispatch, all_excludes} ); // Step 3.3 Consider at_least_one if (format.at_least_one) { std::vector first_choice_elements; for (int it_tag = 0; it_tag < static_cast(format.tags.size()); ++it_tag) { const auto& tag = format.tags[it_tag]; auto begin_expr_id = BuildBeginExpr(tag); auto rule_ref_expr_id = grammar_builder_.AddRuleRef(tag_content_rule_ids[it_tag]); auto end_expr_id = BuildEndExpr(tag); first_choice_elements.push_back( grammar_builder_.AddSequence({begin_expr_id, rule_ref_expr_id, end_expr_id}) ); } auto first_choice_expr_id = grammar_builder_.AddChoices(first_choice_elements); auto first_rule_id = grammar_builder_.AddRuleWithHint("triggered_tags_first", first_choice_expr_id); auto tag_dispatch_rule_id = grammar_builder_.AddRuleWithHint("triggered_tags_sub", rule_expr_id); auto ref_first_rule_expr_id = grammar_builder_.AddRuleRef(first_rule_id); auto ref_tag_dispatch_rule_expr_id = grammar_builder_.AddRuleRef(tag_dispatch_rule_id); auto sequence_expr_id = grammar_builder_.AddSequence({ref_first_rule_expr_id, ref_tag_dispatch_rule_expr_id}); rule_expr_id = grammar_builder_.AddChoices({sequence_expr_id}); } auto rule_id = grammar_builder_.AddRuleWithHint("triggered_tags", rule_expr_id); return ResultOk(rule_id); } Result StructuralTagGrammarConverter::VisitSub(const TagsWithSeparatorFormat& format ) { // The grammar: // Step 1. tags_rule: call tags // tags_rule ::= tag1 | tag2 | ... | tagN // Step 2. Special handling (stop_after_first is true): // if at_least_one is false: // root ::= tags_rule | "" // if at_least_one is true: // root ::= tags_rule // Step 3. Normal handling (stop_after_first is false): // if at_least_one is false: // root ::= tags_rule tags_rule_sub | "" // if at_least_one is true: // root ::= tags_rule tags_rule_sub // tags_rule_sub ::= sep tags_rule tags_rule_sub | "" // Step 1. Construct a rule representing any tag std::vector choice_ids; for (int it_tag = 0; it_tag < static_cast(format.tags.size()); ++it_tag) { auto tag_rule_id = Visit(format.tags[it_tag]); if (tag_rule_id.IsErr()) { return tag_rule_id; } auto tag_rule_ref_id = grammar_builder_.AddRuleRef(std::move(tag_rule_id).Unwrap()); auto sequence_expr_id = grammar_builder_.AddSequence({tag_rule_ref_id}); choice_ids.push_back(sequence_expr_id); } auto choice_expr_id = grammar_builder_.AddChoices(choice_ids); auto all_tags_rule_id = grammar_builder_.AddRuleWithHint("tags_with_separator_tags", choice_expr_id); auto all_tags_rule_ref_id = grammar_builder_.AddRuleRef(all_tags_rule_id); // Step 2. Special case (stop_after_first is true): if (format.stop_after_first) { int32_t rule_body_expr_id; if (format.at_least_one) { // root ::= tags_rule rule_body_expr_id = grammar_builder_.AddChoices({grammar_builder_.AddSequence({all_tags_rule_ref_id})}); } else { // root ::= tags_rule | "" rule_body_expr_id = grammar_builder_.AddChoices( {grammar_builder_.AddSequence({all_tags_rule_ref_id}), grammar_builder_.AddEmptyStr()} ); } auto rule_id = grammar_builder_.AddRuleWithHint("tags_with_separator", rule_body_expr_id); return ResultOk(rule_id); } // Step 3. Normal handling (stop_after_first is false): // Step 3.1 Construct sub rule: sub ::= sep tags sub | "" auto sub_rule_id = grammar_builder_.AddEmptyRuleWithHint("tags_with_separator_sub"); auto end_str_sequence_id = grammar_builder_.AddEmptyStr(); std::vector sub_sequence_elements; if (!format.separator.empty()) { sub_sequence_elements.push_back(grammar_builder_.AddByteString(format.separator)); } sub_sequence_elements.push_back(all_tags_rule_ref_id); sub_sequence_elements.push_back(grammar_builder_.AddRuleRef(sub_rule_id)); auto sub_rule_body_id = grammar_builder_.AddChoices( {grammar_builder_.AddSequence(sub_sequence_elements), end_str_sequence_id} ); grammar_builder_.UpdateRuleBody(sub_rule_id, sub_rule_body_id); // Step 3.2 Construct root rule std::vector choices = { grammar_builder_.AddSequence({all_tags_rule_ref_id, grammar_builder_.AddRuleRef(sub_rule_id)} ), }; if (!format.at_least_one) { choices.push_back(end_str_sequence_id); } auto rule_body_expr_id = grammar_builder_.AddChoices(choices); auto rule_id = grammar_builder_.AddRuleWithHint("tags_with_separator", rule_body_expr_id); return ResultOk(rule_id); } Result StructuralTagGrammarConverter::VisitSub(const OptionalFormat& format) { // optional: 0 or 1 occurrence -> Choice(content, "") auto result = Visit(*format.content); if (result.IsErr()) { return result; } int content_rule_id = std::move(result).Unwrap(); auto content_ref = grammar_builder_.AddRuleRef(content_rule_id); auto expr = grammar_builder_.AddChoices( {grammar_builder_.AddEmptyStr(), grammar_builder_.AddSequence({content_ref})} ); return ResultOk(grammar_builder_.AddRuleWithHint("optional", expr)); } Result StructuralTagGrammarConverter::VisitSub(const PlusFormat& format) { // plus: 1 or more occurrences -> content content_star, where content_star = content content_star // | "" auto result = Visit(*format.content); if (result.IsErr()) { return result; } int content_rule_id = std::move(result).Unwrap(); auto content_ref = grammar_builder_.AddRuleRef(content_rule_id); auto star_rule_id = grammar_builder_.AddEmptyRuleWithHint("plus_star"); auto star_ref = grammar_builder_.AddRuleRef(star_rule_id); auto star_body = grammar_builder_.AddChoices( {grammar_builder_.AddEmptyStr(), grammar_builder_.AddSequence({content_ref, star_ref})} ); grammar_builder_.UpdateRuleBody(star_rule_id, star_body); auto plus_expr = grammar_builder_.AddSequence({content_ref, star_ref}); return ResultOk(grammar_builder_.AddRuleWithHint("plus", plus_expr)); } Result StructuralTagGrammarConverter::VisitSub(const StarFormat& format) { // star: 0 or more occurrences -> content_star, where content_star = content content_star | "" auto result = Visit(*format.content); if (result.IsErr()) { return result; } int content_rule_id = std::move(result).Unwrap(); auto content_ref = grammar_builder_.AddRuleRef(content_rule_id); auto star_rule_id = grammar_builder_.AddEmptyRuleWithHint("star"); auto star_ref = grammar_builder_.AddRuleRef(star_rule_id); auto star_body = grammar_builder_.AddChoices( {grammar_builder_.AddEmptyStr(), grammar_builder_.AddSequence({content_ref, star_ref})} ); grammar_builder_.UpdateRuleBody(star_rule_id, star_body); return ResultOk(grammar_builder_.AddRuleWithHint("star", star_ref)); } Result StructuralTagGrammarConverter::VisitSub(const TokenFormat& format) { XGRAMMAR_DCHECK(format.resolved_token_id_ >= 0) << "TokenFormat must be resolved before conversion"; auto token_set_expr = grammar_builder_.AddTokenSet({format.resolved_token_id_}); auto seq = grammar_builder_.AddSequence({token_set_expr}); auto choices = grammar_builder_.AddChoices({seq}); return ResultOk(grammar_builder_.AddRuleWithHint("token", choices)); } Result StructuralTagGrammarConverter::VisitSub(const ExcludeTokenFormat& format) { std::vector all_excludes = format.resolved_token_ids_; for (auto tid : format.detected_end_token_ids_) { all_excludes.push_back(tid); } int expr = grammar_builder_.AddExcludeTokenSet(all_excludes); auto seq = grammar_builder_.AddSequence({expr}); auto choices = grammar_builder_.AddChoices({seq}); return ResultOk(grammar_builder_.AddRuleWithHint("exclude_token", choices)); } Result StructuralTagGrammarConverter::VisitSub(const AnyTokensFormat& format) { std::vector all_excludes = format.resolved_exclude_token_ids_; for (auto tid : format.detected_end_token_ids_) { all_excludes.push_back(tid); } int exclude_expr = grammar_builder_.AddExcludeTokenSet(all_excludes); int exclude_seq = grammar_builder_.AddSequence({exclude_expr}); int exclude_choices = grammar_builder_.AddChoices({exclude_seq}); int inner_rule = grammar_builder_.AddRuleWithHint("any_tokens_inner", exclude_choices); auto inner_ref = grammar_builder_.AddRuleRef(inner_rule); auto star_rule_id = grammar_builder_.AddEmptyRuleWithHint("any_tokens"); auto star_ref = grammar_builder_.AddRuleRef(star_rule_id); auto star_body = grammar_builder_.AddChoices( {grammar_builder_.AddEmptyStr(), grammar_builder_.AddSequence({inner_ref, star_ref})} ); grammar_builder_.UpdateRuleBody(star_rule_id, star_body); return ResultOk(star_rule_id); } Result StructuralTagGrammarConverter::VisitSub(const TokenTriggeredTagsFormat& format ) { // Step 1. Visit all tags, map trigger → tag IDs std::vector> trigger_to_tag_ids(format.trigger_tokens.size()); std::vector tag_content_rule_ids; tag_content_rule_ids.reserve(format.tags.size()); for (int it_tag = 0; it_tag < static_cast(format.tags.size()); ++it_tag) { const auto& tag = format.tags[it_tag]; if (!std::holds_alternative(tag.begin)) { return ResultErr( "Tags in token_triggered_tags must have a token format begin, not a string" ); } auto begin_token_id = std::get(tag.begin).resolved_token_id_; int matched = -1; for (int it_t = 0; it_t < static_cast(format.resolved_trigger_token_ids_.size()); ++it_t) { if (format.resolved_trigger_token_ids_[it_t] == begin_token_id) { if (matched != -1) { return ResultErr("Tag matches multiple triggers"); } matched = it_t; } } if (matched == -1) { return ResultErr("Tag does not match any trigger"); } trigger_to_tag_ids[matched].push_back(it_tag); auto result = Visit(*tag.content); if (result.IsErr()) return result; tag_content_rule_ids.push_back(std::move(result).Unwrap()); } // Step 2. Special case: at_least_one && stop_after_first if (format.at_least_one && format.stop_after_first) { std::vector choice_elements; for (int it_tag = 0; it_tag < static_cast(format.tags.size()); ++it_tag) { const auto& tag = format.tags[it_tag]; auto begin_expr = BuildBeginExpr(tag); auto ref = grammar_builder_.AddRuleRef(tag_content_rule_ids[it_tag]); auto end_expr = BuildEndExpr(tag); choice_elements.push_back(grammar_builder_.AddSequence({begin_expr, ref, end_expr})); } auto choice = grammar_builder_.AddChoices(choice_elements); return ResultOk(grammar_builder_.AddRuleWithHint("token_triggered_tags", choice)); } // Step 3. Normal case — TokenTagDispatch std::vector> trigger_rule_pairs; for (int it_t = 0; it_t < static_cast(format.trigger_tokens.size()); ++it_t) { std::vector choice_elements; for (auto tag_id : trigger_to_tag_ids[it_t]) { const auto& tag = format.tags[tag_id]; auto ref = grammar_builder_.AddRuleRef(tag_content_rule_ids[tag_id]); auto end_expr = BuildEndExpr(tag); choice_elements.push_back(grammar_builder_.AddSequence({ref, end_expr})); } auto choice = grammar_builder_.AddChoices(choice_elements); auto sub_rule = grammar_builder_.AddRuleWithHint("token_triggered_tags_group", choice); trigger_rule_pairs.push_back({format.resolved_trigger_token_ids_[it_t], sub_rule}); } bool loop = !format.stop_after_first; std::vector all_excludes = format.resolved_exclude_token_ids_; for (auto tid : format.detected_end_token_ids_) { all_excludes.push_back(tid); } auto ttd_expr = grammar_builder_.AddTokenTagDispatch( Grammar::Impl::TokenTagDispatch{trigger_rule_pairs, loop, all_excludes} ); int32_t rule_expr_id = ttd_expr; if (format.at_least_one) { std::vector first_choices; for (int it_tag = 0; it_tag < static_cast(format.tags.size()); ++it_tag) { const auto& tag = format.tags[it_tag]; auto begin_expr = BuildBeginExpr(tag); auto ref = grammar_builder_.AddRuleRef(tag_content_rule_ids[it_tag]); auto end_expr = BuildEndExpr(tag); first_choices.push_back(grammar_builder_.AddSequence({begin_expr, ref, end_expr})); } auto first_choice = grammar_builder_.AddChoices(first_choices); auto first_rule = grammar_builder_.AddRuleWithHint("token_triggered_tags_first", first_choice); auto dispatch_rule = grammar_builder_.AddRuleWithHint("token_triggered_tags_sub", rule_expr_id); auto seq = grammar_builder_.AddSequence( {grammar_builder_.AddRuleRef(first_rule), grammar_builder_.AddRuleRef(dispatch_rule)} ); rule_expr_id = grammar_builder_.AddChoices({seq}); } return ResultOk(grammar_builder_.AddRuleWithHint("token_triggered_tags", rule_expr_id)); } Result StructuralTagGrammarConverter::VisitSub(const RepeatFormat& format) { auto result = Visit(*format.content); if (result.IsErr()) { return result; } int content_rule_id = std::move(result).Unwrap(); int repeat_expr_id = grammar_builder_.AddRepeat(content_rule_id, format.min, format.max); return ResultOk(grammar_builder_.AddRuleWithHint("repeat", repeat_expr_id)); } Result StructuralTagGrammarConverter::VisitSub(const DispatchFormat& format) { std::vector> tag_rule_pairs; tag_rule_pairs.reserve(format.rules.size()); for (const auto& pair : format.rules) { if (!pair.second) { return ResultErr("TagDispatch pair must have content"); } auto result = Visit(*pair.second); if (result.IsErr()) { return result; } tag_rule_pairs.push_back({pair.first, std::move(result).Unwrap()}); } auto rule_expr_id = grammar_builder_.AddTagDispatch( Grammar::Impl::TagDispatch{std::move(tag_rule_pairs), format.loop, format.excludes} ); return ResultOk(grammar_builder_.AddRuleWithHint("tag_dispatch", rule_expr_id)); } Result StructuralTagGrammarConverter::VisitSub(const TokenDispatchFormat& format) { XGRAMMAR_DCHECK(format.resolved_trigger_token_ids_.size() == format.rules.size()) << "TokenDispatchFormat must be resolved before conversion"; std::vector> trigger_rule_pairs; trigger_rule_pairs.reserve(format.rules.size()); for (size_t i = 0; i < format.rules.size(); ++i) { const auto& pair = format.rules[i]; if (!pair.second) { return ResultErr("TokenTagDispatch pair must have content"); } auto result = Visit(*pair.second); if (result.IsErr()) { return result; } trigger_rule_pairs.push_back({format.resolved_trigger_token_ids_[i], std::move(result).Unwrap()} ); } std::vector all_excludes = format.resolved_exclude_token_ids_; auto rule_expr_id = grammar_builder_.AddTokenTagDispatch( Grammar::Impl::TokenTagDispatch{trigger_rule_pairs, format.loop, all_excludes} ); return ResultOk(grammar_builder_.AddRuleWithHint("token_tag_dispatch", rule_expr_id)); } /************** StructuralTag Conversion Public API **************/ Result StructuralTagToGrammar( const std::string& structural_tag_json, const std::optional& tokenizer_info ) { auto structural_tag_result = StructuralTagParser::FromJSON(structural_tag_json); if (structural_tag_result.IsErr()) { return ResultErr(std::move(structural_tag_result).UnwrapErr()); } auto structural_tag = std::move(structural_tag_result).Unwrap(); auto resolve_err = StructuralTagTokenResolver::Resolve(&structural_tag, tokenizer_info); if (resolve_err.has_value()) { return ResultErr(std::move(resolve_err).value()); } auto err = StructuralTagAnalyzer().Analyze(&structural_tag); if (err.has_value()) { return ResultErr(std::move(err).value()); } auto result = StructuralTagGrammarConverter::Convert(structural_tag); if (result.IsErr()) { return ResultErr(std::move(result).UnwrapErr()); } return ResultOk(GrammarNormalizer::Apply(std::move(result).Unwrap())); } } // namespace xgrammar xgrammar-0.2.3/cpp/structural_tag.h000066400000000000000000000276301521764210300173410ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/structural_tag_impl.h * \brief The implementation header for the structural tag. */ #ifndef XGRAMMAR_STRUCTURAL_TAG_H_ #define XGRAMMAR_STRUCTURAL_TAG_H_ #include #include #include #include #include #include #include #include #include #include #include "support/utils.h" #include "xgrammar/tokenizer_info.h" namespace xgrammar { /******************** Structural Tag Definition ********************/ // TODO(yixin): Consider moving the definition to Public API. struct ConstStringFormat; struct JSONSchemaFormat; struct AnyTextFormat; struct GrammarFormat; struct RegexFormat; struct SequenceFormat; struct OrFormat; struct TagFormat; struct TriggeredTagsFormat; struct TagsWithSeparatorFormat; struct OptionalFormat; struct PlusFormat; struct StarFormat; struct TokenFormat; struct ExcludeTokenFormat; struct AnyTokensFormat; struct TokenTriggeredTagsFormat; struct RepeatFormat; struct DispatchFormat; struct TokenDispatchFormat; using Format = std::variant< ConstStringFormat, JSONSchemaFormat, AnyTextFormat, GrammarFormat, RegexFormat, SequenceFormat, OrFormat, TagFormat, TriggeredTagsFormat, TagsWithSeparatorFormat, OptionalFormat, PlusFormat, StarFormat, TokenFormat, ExcludeTokenFormat, AnyTokensFormat, TokenTriggeredTagsFormat, RepeatFormat, DispatchFormat, TokenDispatchFormat>; /******************** Basic Formats ********************/ struct ConstStringFormat { static constexpr const char* type = "const_string"; std::string value; ConstStringFormat(std::string value) : value(std::move(value)) {} picojson::value ToJSON() const; }; struct JSONSchemaFormat { static constexpr const char* type = "json_schema"; std::string json_schema; std::string style = "json"; // "json","qwen_xml","minimax_xml","deepseek_xml","glm_xml" // Whether to allow object properties to appear in any order. See // Grammar::FromJSONSchema / JSONSchemaToEBNF for the semantics. bool any_order = false; JSONSchemaFormat(std::string json_schema, std::string style = "json", bool any_order = false) : json_schema(std::move(json_schema)), style(std::move(style)), any_order(any_order) {} picojson::value ToJSON() const; }; struct GrammarFormat { static constexpr const char* type = "grammar"; std::string grammar; GrammarFormat(std::string grammar) : grammar(std::move(grammar)) {} picojson::value ToJSON() const; }; struct RegexFormat { static constexpr const char* type = "regex"; std::string pattern; RegexFormat(std::string pattern) : pattern(std::move(pattern)) {} picojson::value ToJSON() const; }; struct AnyTextFormat { static constexpr const char* type = "any_text"; std::vector excludes; AnyTextFormat(std::vector excluded_strs) : excludes(std::move(excluded_strs)) {} picojson::value ToJSON() const; private: std::vector detected_end_strs_; friend class StructuralTagAnalyzer; friend class StructuralTagGrammarConverter; }; struct TokenFormat { static constexpr const char* type = "token"; std::variant token; TokenFormat(std::variant token) : token(std::move(token)) { if (std::holds_alternative(this->token)) { resolved_token_id_ = std::get(this->token); } } picojson::value ToJSON() const; private: int32_t resolved_token_id_ = -1; friend class StructuralTagTokenResolver; friend class StructuralTagAnalyzer; friend class StructuralTagGrammarConverter; }; struct ExcludeTokenFormat { static constexpr const char* type = "exclude_token"; std::vector> exclude_tokens; ExcludeTokenFormat(std::vector> exclude_tokens) : exclude_tokens(std::move(exclude_tokens)) {} picojson::value ToJSON() const; private: std::vector resolved_token_ids_; std::vector detected_end_token_ids_; friend class StructuralTagTokenResolver; friend class StructuralTagAnalyzer; friend class StructuralTagGrammarConverter; }; struct AnyTokensFormat { static constexpr const char* type = "any_tokens"; std::vector> exclude_tokens; AnyTokensFormat(std::vector> exclude_tokens) : exclude_tokens(std::move(exclude_tokens)) {} picojson::value ToJSON() const; private: std::vector resolved_exclude_token_ids_; std::vector detected_end_token_ids_; friend class StructuralTagTokenResolver; friend class StructuralTagAnalyzer; friend class StructuralTagGrammarConverter; }; /******************** Combinatorial Formats ********************/ struct SequenceFormat { static constexpr const char* type = "sequence"; std::vector elements; SequenceFormat(std::vector elements); picojson::value ToJSON() const; private: // Detected in StructuralTagAnalyzer bool is_unlimited_ = false; friend class StructuralTagAnalyzer; friend class StructuralTagGrammarConverter; }; struct OrFormat { static constexpr const char* type = "or"; std::vector elements; OrFormat(std::vector elements); picojson::value ToJSON() const; private: // Detected in StructuralTagAnalyzer bool is_unlimited_ = false; friend class StructuralTagAnalyzer; friend class StructuralTagGrammarConverter; }; struct TagFormat { static constexpr const char* type = "tag"; std::variant begin; std::shared_ptr content; std::variant, TokenFormat> end; TagFormat( std::variant begin, std::shared_ptr content, std::variant, TokenFormat> end ) : begin(std::move(begin)), content(std::move(content)), end(std::move(end)) {} picojson::value ToJSON() const; }; struct TriggeredTagsFormat { static constexpr const char* type = "triggered_tags"; std::vector triggers; std::vector tags; std::vector excludes; bool at_least_one = false; bool stop_after_first = false; TriggeredTagsFormat( std::vector triggers, std::vector tags, std::vector excludes, bool at_least_one, bool stop_after_first ) : triggers(std::move(triggers)), tags(std::move(tags)), excludes(std::move(excludes)), at_least_one(at_least_one), stop_after_first(stop_after_first) {} picojson::value ToJSON() const; private: std::vector detected_end_strs_; friend class StructuralTagAnalyzer; friend class StructuralTagGrammarConverter; }; struct TagsWithSeparatorFormat { static constexpr const char* type = "tags_with_separator"; std::vector tags; std::string separator; bool at_least_one = false; bool stop_after_first = false; TagsWithSeparatorFormat( std::vector tags, std::string separator, bool at_least_one, bool stop_after_first ) : tags(std::move(tags)), separator(std::move(separator)), at_least_one(at_least_one), stop_after_first(stop_after_first) {} picojson::value ToJSON() const; private: friend class StructuralTagAnalyzer; friend class StructuralTagGrammarConverter; }; struct TokenTriggeredTagsFormat { static constexpr const char* type = "token_triggered_tags"; std::vector> trigger_tokens; std::vector tags; std::vector> exclude_tokens; bool at_least_one = false; bool stop_after_first = false; TokenTriggeredTagsFormat( std::vector> trigger_tokens, std::vector tags, std::vector> exclude_tokens, bool at_least_one, bool stop_after_first ) : trigger_tokens(std::move(trigger_tokens)), tags(std::move(tags)), exclude_tokens(std::move(exclude_tokens)), at_least_one(at_least_one), stop_after_first(stop_after_first) {} picojson::value ToJSON() const; private: std::vector resolved_trigger_token_ids_; std::vector resolved_exclude_token_ids_; std::vector detected_end_token_ids_; friend class StructuralTagTokenResolver; friend class StructuralTagAnalyzer; friend class StructuralTagGrammarConverter; }; struct OptionalFormat { static constexpr const char* type = "optional"; std::shared_ptr content; OptionalFormat(std::shared_ptr content) : content(std::move(content)) {} picojson::value ToJSON() const; }; struct PlusFormat { static constexpr const char* type = "plus"; std::shared_ptr content; PlusFormat(std::shared_ptr content) : content(std::move(content)) {} picojson::value ToJSON() const; }; struct StarFormat { static constexpr const char* type = "star"; std::shared_ptr content; StarFormat(std::shared_ptr content) : content(std::move(content)) {} picojson::value ToJSON() const; }; struct RepeatFormat { static constexpr const char* type = "repeat"; int32_t min; int32_t max; std::shared_ptr content; RepeatFormat(int32_t min, int32_t max, std::shared_ptr content) : min(min), max(max), content(std::move(content)) {} picojson::value ToJSON() const; }; /*! * \brief A format that maps directly to a TagDispatch grammar. * Accepts ``[trigger string, content format]`` pairs in JSON; each content is converted to a rule * and the result is a single TagDispatch(loop, excludes). */ struct DispatchFormat { static constexpr const char* type = "dispatch"; std::vector>> rules; bool loop = true; std::vector excludes; DispatchFormat( std::vector>> rules, bool loop = true, std::vector excludes = {} ) : rules(std::move(rules)), loop(loop), excludes(std::move(excludes)) {} picojson::value ToJSON() const; }; /*! * \brief A format that maps directly to a TokenTagDispatch grammar. * Accepts ``[trigger token, content format]`` pairs in JSON; trigger can be token ID or token * string (resolved via tokenizer_info). Each content is converted to a rule. */ struct TokenDispatchFormat { static constexpr const char* type = "token_dispatch"; std::vector, std::shared_ptr>> rules; bool loop = true; std::vector> exclude_tokens; TokenDispatchFormat( std::vector, std::shared_ptr>> rules, bool loop = true, std::vector> exclude_tokens = {} ) : rules(std::move(rules)), loop(loop), exclude_tokens(std::move(exclude_tokens)) {} picojson::value ToJSON() const; private: std::vector resolved_trigger_token_ids_; std::vector resolved_exclude_token_ids_; friend class StructuralTagTokenResolver; friend class StructuralTagGrammarConverter; }; /******************** Top Level ********************/ struct StructuralTag { static constexpr const char* type = "structural_tag"; Format format; StructuralTag(Format format) : format(std::move(format)) {} }; /******************** Conversion API ********************/ /*! * \brief Convert a structural tag JSON string to a grammar. * \param structural_tag_json The JSON string of the structural tag. * \return A grammar if the JSON is valid, otherwise an error message in std::string. */ Result StructuralTagToGrammar( const std::string& structural_tag_json, const std::optional& tokenizer_info = std::nullopt ); } // namespace xgrammar #endif // XGRAMMAR_STRUCTURAL_TAG_H_ xgrammar-0.2.3/cpp/support/000077500000000000000000000000001521764210300156315ustar00rootroot00000000000000xgrammar-0.2.3/cpp/support/compact_2d_array.h000066400000000000000000000305161521764210300212200ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/support/compact_2d_array.h */ #ifndef XGRAMMAR_SUPPORT_COMPACT_2D_ARRAY_H_ #define XGRAMMAR_SUPPORT_COMPACT_2D_ARRAY_H_ #include #include #include #include #include #include "logging.h" #include "memory_size.h" #include "reflection.h" namespace xgrammar { /*! * \brief This class implements a Compressed Sparse Row (CSR) array data structure. It stores * a 2D array in a compressed format, where each row can have a variable number of elements, and * all rows are stored contiguously in memory. The inserted row is immutable. * * \note Inserting new rows into the Compact2DArray will invalidate the existing Row objects. * * \tparam DataType The type of elements stored in the Compact2DArray. * * \details * The Compact2DArray stores elements of type DataType in a compressed format, * where each row can have a variable number of elements. It uses two vectors: * - data_: stores all elements contiguously * - indptr_: stores the starting index of each row in data_. Its last element is the size of data_ * representing the ending index. * * This structure allows efficient storage and access for sparse data. */ template class Compact2DArray { public: /*! * \brief The struct representing a row in the Compact2DArray. */ struct Row { /*! \brief The value type is DataType. */ using value_type = DataType; /*! \brief Pointer to the data of the row. */ const DataType* data; /*! \brief Length of the row data. */ int32_t data_len; /*! * \brief Access an element in the row. * \param i Index of the element to access. * \return Reference to the element at index i. */ const DataType& operator[](int32_t i) const { XGRAMMAR_DCHECK(i >= 0 && i < data_len) << "Index " << i << " of the Compact2DArray Row is out of bound"; return data[i]; } /*! \brief Get the beginning iterator of the row. */ const DataType* begin() const { return data; } /*! \brief Get the end iterator of the row. */ const DataType* end() const { return data + data_len; } /*! \brief Get the size of the row. */ int32_t size() const { return data_len; } /*! \brief Get a sub-row in [begin, end). */ Row Slice(int32_t begin, int32_t end) const { XGRAMMAR_DCHECK(begin >= 0 && begin <= end && end <= data_len) << "Compact2DArray Row slice is out of bound"; return {data + begin, end - begin}; } friend std::ostream& operator<<(std::ostream& os, const Row& row) { os << "["; for (auto i = 0; i < row.data_len; ++i) { if (i > 0) { os << ", "; } os << row[i]; } os << "]"; return os; } }; /*! * \brief The mutable struct representing a row in the Compact2DArray. */ struct MutableRow { /*! \brief The value type is DataType. */ using value_type = DataType; /*! \brief Pointer to the data of the row. */ DataType* data; /*! \brief Length of the row data. */ int32_t data_len; /*! * \brief Access an element in the row. * \param i Index of the element to access. * \return Reference to the element at index i. */ DataType& operator[](int32_t i) const { XGRAMMAR_DCHECK(i >= 0 && i < data_len) << "Index " << i << " of the Compact2DArray MutableRow is out of bound"; return data[i]; } /*! \brief Get the beginning iterator of the row. */ DataType* begin() const { return data; } /*! \brief Get the end iterator of the row. */ DataType* end() const { return data + data_len; } /*! \brief Get the size of the row. */ int32_t size() const { return data_len; } }; /*! \brief The value type is Row. */ using value_type = Row; /*! \brief Default constructor. */ Compact2DArray() = default; /*! * \brief Construct a Compact2DArray from an existing CSR representation. * \param data All row elements stored contiguously. * \param indptr Row start offsets. Must start with 0, be non-decreasing, and end with * data.size(). * \return The constructed Compact2DArray. */ static Compact2DArray FromDataAndIndptr(std::vector data, std::vector indptr); /*! * \brief Construct a Compact2DArray from row sizes with default-constructed data. * \param row_sizes The size of each row. * \return The constructed Compact2DArray. */ static Compact2DArray FromRowSizes(const std::vector& row_sizes); /*! * \brief Reset the Compact2DArray from row sizes with default-constructed data. * \param row_sizes The size of each row. */ void ResetWithRowSizes(const std::vector& row_sizes); /****************** Accessors ******************/ /*! \brief Get the number of rows in the Compact2DArray. */ int32_t size() const { return static_cast(indptr_.size()) - 1; } friend std::size_t MemorySize(const Compact2DArray& arr) { return MemorySize(arr.data_) + MemorySize(arr.indptr_); } /*! * \brief Access a row in the Compact2DArray. * \param i Index of the row to access. * \return Row struct representing the i-th row. */ Row operator[](int32_t i) const; /*! * \brief Access a mutable row in the Compact2DArray. * \param i Index of the row to access. * \return MutableRow struct representing the i-th row. */ MutableRow MutableRowAt(int32_t i); /****************** Modifiers ******************/ /*! * \brief Insert a new row of data into the Compact2DArray. * \param data Pointer to the data to be inserted. * \param data_len Length of the data to be inserted. * \return The index of the newly inserted row. */ int32_t PushBack(const DataType* new_data, int32_t new_data_len); /*! * \brief Insert a new row of data into the Compact2DArray from a vector. * \param data Vector containing the data to be inserted. * \return The index of the newly inserted row. */ int32_t PushBack(const std::vector& new_data); /*! * \brief Insert a new row of data into the Compact2DArray from a Row struct. * \param row The Row struct containing the data to be inserted. * \return The index of the newly inserted row. */ int32_t PushBack(const Row& row) { return PushBack(row.data, row.data_len); } /*! * \brief Push back a new element in the latest row. * \param new_data the element to be pushed. */ void PushBackInLatestRow(const DataType& new_data) { XGRAMMAR_DCHECK(!indptr_.empty()) << "Cannot push back in an empty Compact2DArray"; data_.push_back(new_data); indptr_.back()++; } Row Back() { return (*this)[size() - 1]; } /*! * \brief Insert a new row of non-contiguous data into the Compact2DArray. This method inserts a * single element followed by a sequence of elements. This is useful in the GrammarExpr data * structure. * \param data_1 The first element to be inserted. * \param data_2 Pointer to the remaining data to be inserted. * \param data_2_len Length of the remaining data to be inserted. * \return The index of the newly inserted row. */ int32_t PushBackNonContiguous(DataType data_1, const DataType* data_2, int32_t data_2_len); /*! * \brief Pop back the last one or multiple rows of the Compact2DArray. * \param cnt The number of rows to be popped. */ void PopBack(const int32_t& cnt) { indptr_.erase(indptr_.end() - cnt, indptr_.end()); data_.erase(data_.begin() + indptr_.back(), data_.end()); return; } /****************** Internal Accessors ******************/ /*! \brief Get a pointer to the underlying data array. */ const DataType* data() const { return data_.data(); } /*! \brief Get a pointer to the underlying index pointer array. */ const int32_t* indptr() const { return indptr_.data(); } /****************** Printing ******************/ friend std::ostream& operator<<(std::ostream& os, const Compact2DArray& compact_2d_array) { os << "Compact2DArray(["; for (auto i = 0; i < compact_2d_array.size(); ++i) { if (i > 0) { os << ", "; } os << compact_2d_array[i]; } os << "])"; return os; } private: /*! \brief Vector storing all elements contiguously. */ std::vector data_; /*! \brief Vector storing the starting index of each row in data_. */ std::vector indptr_{0}; friend struct member_trait>; }; template inline typename Compact2DArray::Row Compact2DArray::operator[](int32_t i ) const { XGRAMMAR_DCHECK(i >= 0 && i < size()) << "Compact2DArray index " << i << " is out of bound"; int32_t start = indptr_[i]; int32_t end = indptr_[i + 1]; return {data_.data() + start, end - start}; } template inline typename Compact2DArray::MutableRow Compact2DArray::MutableRowAt( int32_t i ) { XGRAMMAR_DCHECK(i >= 0 && i < size()) << "Compact2DArray index " << i << " is out of bound"; int32_t start = indptr_[i]; int32_t end = indptr_[i + 1]; return {data_.data() + start, end - start}; } template inline Compact2DArray Compact2DArray::FromDataAndIndptr( std::vector data, std::vector indptr ) { XGRAMMAR_CHECK(!indptr.empty()) << "Compact2DArray indptr cannot be empty"; XGRAMMAR_CHECK(indptr.front() == 0) << "Compact2DArray indptr must start with 0"; for (int32_t i = 1; i < static_cast(indptr.size()); ++i) { XGRAMMAR_CHECK(indptr[i - 1] <= indptr[i]) << "Compact2DArray indptr must be non-decreasing"; } XGRAMMAR_CHECK(indptr.back() == static_cast(data.size())) << "Compact2DArray indptr must end with data.size()"; Compact2DArray result; result.data_ = std::move(data); result.indptr_ = std::move(indptr); return result; } template inline Compact2DArray Compact2DArray::FromRowSizes( const std::vector& row_sizes ) { Compact2DArray result; result.ResetWithRowSizes(row_sizes); return result; } template inline void Compact2DArray::ResetWithRowSizes(const std::vector& row_sizes) { indptr_.resize(row_sizes.size() + 1); indptr_[0] = 0; for (int32_t i = 0; i < static_cast(row_sizes.size()); ++i) { XGRAMMAR_CHECK(row_sizes[i] >= 0) << "Compact2DArray row size cannot be negative"; indptr_[i + 1] = indptr_[i] + row_sizes[i]; } data_.resize(indptr_.back()); } template inline int32_t Compact2DArray::PushBack(const DataType* new_data, int32_t new_data_len) { // TODO(yixin): whether to add a additional data_len // If the new data is already in the Compact2DArray, we need to copy it to the new memory // location. if (new_data >= data_.data() && new_data < data_.data() + data_.size()) { std::vector new_data_copied(new_data, new_data + new_data_len); data_.insert(data_.end(), new_data_copied.begin(), new_data_copied.end()); } else { data_.insert(data_.end(), new_data, new_data + new_data_len); } indptr_.push_back(static_cast(data_.size())); return static_cast(indptr_.size()) - 2; } template inline int32_t Compact2DArray::PushBack(const std::vector& new_data) { data_.insert(data_.end(), new_data.begin(), new_data.end()); indptr_.push_back(static_cast(data_.size())); return static_cast(indptr_.size()) - 2; } template inline int32_t Compact2DArray::PushBackNonContiguous( DataType data_1, const DataType* data_2, int32_t data_2_len ) { if (data_2 >= data_.data() && data_2 < data_.data() + data_.size()) { std::vector new_data_copied(data_2, data_2 + data_2_len); data_.push_back(data_1); data_.insert(data_.end(), new_data_copied.begin(), new_data_copied.end()); } else { data_.push_back(data_1); data_.insert(data_.end(), data_2, data_2 + data_2_len); } indptr_.push_back(static_cast(data_.size())); return static_cast(indptr_.size()) - 2; } template XGRAMMAR_MEMBER_TABLE_TEMPLATE( Compact2DArray, "data_", &Compact2DArray::data_, "indptr_", &Compact2DArray::indptr_ ); } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_COMPACT_2D_ARRAY_H_ xgrammar-0.2.3/cpp/support/container.h000066400000000000000000000074121521764210300177700ustar00rootroot00000000000000/*! * Copyright (c) 2025 by Contributors * \file xgrammar/support/container.h * \brief The header for container. */ #ifndef XGRAMMAR_SUPPORT_CONTAINER_H_ #define XGRAMMAR_SUPPORT_CONTAINER_H_ #include #include "logging.h" namespace xgrammar { namespace details { template class NodePool { public: NodePool() = default; void Reserve(int n) { node_pool_.reserve(n); } [[nodiscard]] int Allocate() { if (free_list_.empty()) { int node = Size(); node_pool_.emplace_back(); return node; } else { int node = free_list_.back(); free_list_.pop_back(); return node; } } void Deallocate(int node) { free_list_.push_back(node); } void Clear() { node_pool_.clear(); free_list_.clear(); } Node& operator[](int node) { XGRAMMAR_DCHECK(0 <= node && node < Size()); return node_pool_[node]; } int Size() const { return static_cast(node_pool_.size()); } private: std::vector node_pool_; std::vector free_list_; }; } // namespace details template class List { private: struct Node { int prev; int next; Value value; }; public: struct iterator { public: iterator(int n, List& c) : node_(n), list_(&c) { XGRAMMAR_DCHECK(0 <= node_ && node_ < list_->node_pool_.Size()); } iterator& operator++() { node_ = GetNode().next; return *this; } iterator operator++(int) { iterator tmp = *this; ++*this; return tmp; } Value& operator*() const { return GetNode().value; } Value* operator->() const { return &GetNode().value; } bool operator==(const iterator& rhs) const { XGRAMMAR_DCHECK(list_ == rhs.list_) << "compare different container is UB"; return node_ == rhs.node_; // compare different container is UB } bool operator!=(const iterator& rhs) const { XGRAMMAR_DCHECK(list_ == rhs.list_) << "compare different container is UB"; return node_ != rhs.node_; // compare different container is UB } int Index() const { return node_; } private: friend class List; Node& GetNode() const { return list_->node_pool_[node_]; } int node_; List* list_; }; List(int reserved = 0) { node_pool_.Reserve(reserved); InitGuard(); } iterator PushBack(const Value& value) { int node = node_pool_.Allocate(); XGRAMMAR_DCHECK(0 < node && node < node_pool_.Size()); node_pool_[node].value = value; LinkBefore(node, 0); return iterator(node, *this); } void MoveBack(int node) { XGRAMMAR_DCHECK(0 < node && node < node_pool_.Size()); Unlink(node); LinkBefore(node, 0); } iterator Erase(iterator it) { int node = it.Index(); XGRAMMAR_DCHECK(0 < node && node < node_pool_.Size()); int next = node_pool_[node].next; Unlink(node); node_pool_.Deallocate(node); return iterator(next, *this); } void Clear() { node_pool_.Clear(); InitGuard(); } iterator begin() { return iterator(node_pool_[0].next, *this); } iterator end() { return iterator(0, *this); } private: void InitGuard() { int node_id = node_pool_.Allocate(); XGRAMMAR_DCHECK(node_id == 0) << "node 0 should be reserved as guard node"; node_pool_[0].prev = 0; node_pool_[0].next = 0; } void LinkBefore(int node, int next) { int prev = node_pool_[next].prev; node_pool_[node].prev = prev; node_pool_[node].next = next; node_pool_[prev].next = node; node_pool_[next].prev = node; } void Unlink(int node) { int prev = node_pool_[node].prev; int next = node_pool_[node].next; node_pool_[prev].next = next; node_pool_[next].prev = prev; } details::NodePool node_pool_; }; } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_CONTAINER_H_ xgrammar-0.2.3/cpp/support/cpptrace.h000066400000000000000000000020361521764210300176040ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/support/cpptrace.h * \details This file is an encapsulation of the cpptrace library. It helps debugging. This file * takes effect only when XGRAMMAR_ENABLE_CPPTRACE is set to 1, and only support Linux and * RelWithDebugInfo or Debug build. */ #ifndef XGRAMMAR_SUPPORT_CPPTRACE_H_ #define XGRAMMAR_SUPPORT_CPPTRACE_H_ #if XGRAMMAR_ENABLE_CPPTRACE == 1 #include #endif #include namespace xgrammar { #if XGRAMMAR_ENABLE_CPPTRACE == 1 // Flag to check if cpptrace feature is enabled static constexpr bool CPPTRACE_ENABLED = true; inline void PrintTrace() { cpptrace::generate_trace().print(); } inline std::string GetTraceString() { return cpptrace::generate_trace().to_string(true); } #else static constexpr bool CPPTRACE_ENABLED = false; // Provide empty implementation when cpptrace is disabled inline void PrintTrace() {} inline std::string GetTraceString() { return ""; } #endif } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_CPPTRACE_H_ xgrammar-0.2.3/cpp/support/dynamic_bitset.h000066400000000000000000000260511521764210300210040ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/support/dynamic_bitset.h * \brief The header for utilities used in grammar-guided generation. */ #ifndef XGRAMMAR_SUPPORT_DYNAMIC_BITSET_H_ #define XGRAMMAR_SUPPORT_DYNAMIC_BITSET_H_ #include #include #include #include #include #include // For __popcnt #ifdef _MSC_VER #include #endif #include "json_serializer.h" #include "logging.h" namespace xgrammar { /*! * \brief A bitset whose length is specified at runtime. Note the size cannot be changed after * construction. * \details The buffer of the bitset is a uint32_t array. There are two uses for this class: * - When passing nullptr to data, it maintains an internal buffer for the bitset. * - When passing a pointer to a buffer with enough size, it uses the external buffer for the * bitset. * \details Part of the implementation is adopted from Boost::dynamic_bitset. */ class DynamicBitset { public: /*! * \brief Calculate the minimal size of the uint32_t buffer for the bitset with the given size. * \param element_size The size of the bitset. * \return The minimal buffer size. */ static int GetBufferSize(int element_size) { return (element_size + 31) / 32; } /*! * \brief Construct a empty bitset. This object should be assigned to a valid bitset before using. */ DynamicBitset() : size_(0), buffer_size_(0), data_(nullptr), is_internal_(true) {} /*! * \brief Construct a bitset with the given size. * \param size The size of the bitset. * \param data The buffer for the bitset. If nullptr, the bitset will maintain an internal buffer. */ DynamicBitset(int size, uint32_t* data = nullptr) : size_(size), buffer_size_(GetBufferSize(size)) { if (data == nullptr) { internal_buffer_.resize(buffer_size_, 0); data_ = internal_buffer_.data(); is_internal_ = true; } else { data_ = data; is_internal_ = false; } } /*! \brief Copy constructor. Copy the buffer and manage the memory internally. */ DynamicBitset(const DynamicBitset& other) : size_(other.size_), buffer_size_(other.buffer_size_), data_(), internal_buffer_(), is_internal_(other.is_internal_) { if (other.is_internal_) { // copy the internal buffer internal_buffer_ = other.internal_buffer_; data_ = internal_buffer_.data(); } else { // simply point to the same external buffer data_ = other.data_; } } /*! \brief Move constructor. Reset other and take ownership of its buffer. */ DynamicBitset(DynamicBitset&& other) noexcept : size_(std::exchange(other.size_, 0)), buffer_size_(std::exchange(other.buffer_size_, 0)), data_(std::exchange(other.data_, nullptr)), internal_buffer_(std::move(other.internal_buffer_)), is_internal_(std::exchange(other.is_internal_, true)) {} /*! \brief Copy assignment. */ DynamicBitset& operator=(const DynamicBitset& other) { XGRAMMAR_DCHECK(is_internal_ || size_ >= other.size_) << "Expanding bitset size is not allowed when the " "memory of the bitset is externally managed"; size_ = other.size_; buffer_size_ = other.buffer_size_; if (is_internal_) { internal_buffer_.resize(buffer_size_); data_ = internal_buffer_.data(); } if (data_ != other.data_) { std::memcpy(data_, other.data_, buffer_size_ * sizeof(uint32_t)); } return *this; } /*! \brief Move assignment. */ DynamicBitset& operator=(DynamicBitset&& other) noexcept { size_ = other.size_; buffer_size_ = other.buffer_size_; is_internal_ = other.is_internal_; if (is_internal_) { internal_buffer_ = std::move(other.internal_buffer_); data_ = internal_buffer_.data(); } else { data_ = other.data_; } return *this; } /*! \brief Get the value of the bit at the given index. */ bool operator[](int index) const { XGRAMMAR_DCHECK(data_ && index >= 0 && index < size_); return (data_[index / 32] >> (index % 32)) & 1; } /*! \brief Get the size of the bitset. */ int Size() const { return size_; } /*! \brief Set the whole bitset to true. */ void Set() { XGRAMMAR_DCHECK(data_); std::memset(data_, 0xFF, buffer_size_ * sizeof(uint32_t)); } /*! \brief Set the bit at the given index to the given value. */ void Set(int index, bool value = true) { XGRAMMAR_DCHECK(data_ && index >= 0 && index < size_); if (value) { data_[index / 32] |= 1 << (index % 32); } else { data_[index / 32] &= ~(1 << (index % 32)); } } /*! \brief Set the whole bitset to false. */ void Reset() { XGRAMMAR_DCHECK(data_); std::memset(data_, 0, buffer_size_ * sizeof(uint32_t)); } /*! \brief Set the bit at the given index to false. */ void Reset(int index) { Set(index, false); } /*! \brief Perform a bitwise OR operation between the current bitset and another bitset. */ DynamicBitset& operator|=(const DynamicBitset& other) { XGRAMMAR_DCHECK(buffer_size_ <= other.buffer_size_); for (int i = 0; i < buffer_size_; ++i) { data_[i] |= other.data_[i]; } return *this; } int FindFirstOne() const { return DoFindOneFrom(0); } int FindNextOne(int pos) const { if (pos >= size_ - 1 || size_ == 0) return -1; ++pos; int blk = pos / BITS_PER_BLOCK; int ind = pos % BITS_PER_BLOCK; uint32_t fore = data_[blk] >> ind; int result = fore ? pos + LowestBit(fore) : DoFindOneFrom(blk + 1); return result < size_ ? result : -1; } int FindFirstZero() const { return DoFindZeroFrom(0); } int FindNextZero(int pos) const { if (pos >= size_ - 1 || size_ == 0) return -1; ++pos; int blk = pos / BITS_PER_BLOCK; int ind = pos % BITS_PER_BLOCK; uint32_t fore = (~data_[blk]) >> ind; int result = fore ? pos + LowestBit(fore) : DoFindZeroFrom(blk + 1); return result < size_ ? result : -1; } int Count() const { int count = 0; for (int i = 0; i < buffer_size_; ++i) { count += PopCount(data_[i]); } return count; } bool All() const { if (size_ == 0) return true; // Check all complete blocks except the last one for (int i = 0; i < buffer_size_ - 1; ++i) { if (data_[i] != ~static_cast(0)) { return false; } } // For the last block, create a mask for valid bits only int remaining_bits = size_ % BITS_PER_BLOCK; uint32_t last_block_mask = remaining_bits ? (static_cast(1) << remaining_bits) - 1 : ~static_cast(0); return (data_[buffer_size_ - 1] & last_block_mask) == last_block_mask; } static constexpr int BITS_PER_BLOCK = 32; friend std::size_t MemorySize(const DynamicBitset& bitset) { return bitset.buffer_size_ * sizeof(bitset.data_[0]); } friend picojson::value SerializeJSONValue(const DynamicBitset& bitset) { XGRAMMAR_DCHECK(bitset.buffer_size_ == GetBufferSize(bitset.size_)); picojson::array result; result.reserve(2 + bitset.buffer_size_); result.emplace_back(picojson::value(static_cast(bitset.size_))); result.emplace_back(picojson::value(static_cast(bitset.buffer_size_))); for (int i = 0; i < bitset.buffer_size_; ++i) { result.emplace_back(picojson::value(static_cast(bitset.data_[i]))); } return picojson::value(std::move(result)); } friend std::optional DeserializeJSONValue( DynamicBitset* bitset, const picojson::value& value, const std::string& type_name ) { if (!value.is()) { return ConstructDeserializeError("Expect an array", type_name); } const auto& arr = value.get(); if (arr.size() < 2) { return ConstructDeserializeError("Except at least 2 elements in the array", type_name); } if (!arr[0].is()) { return ConstructDeserializeError("Expect an integer for size", type_name); } int size = static_cast(arr[0].get()); if (!arr[1].is()) { return ConstructDeserializeError("Expect an integer for buffer_size", type_name); } int buffer_size = static_cast(arr[1].get()); if (buffer_size != GetBufferSize(size)) { return ConstructDeserializeError( "Invalid buffer_size. Buffer size should be ceil(size / 32)", type_name ); } DynamicBitset result(size); for (int i = 0; i < buffer_size; ++i) { if (!arr[i + 2].is()) { return ConstructDeserializeError("Expect an integer in the array", type_name); } int64_t value = arr[i + 2].get(); if (value < 0 || value > std::numeric_limits::max()) { return ConstructDeserializeError( "Integer in the array is " + std::to_string(value) + " and out of the uint32_t range", type_name ); } result.data_[i] = static_cast(value); } *bitset = std::move(result); return std::nullopt; } bool operator==(const DynamicBitset& other) const { if (size_ != other.size_) return false; if (buffer_size_ != other.buffer_size_) return false; for (int i = 0; i < buffer_size_; ++i) { if (data_[i] != other.data_[i]) return false; } return true; } private: static int LowestBit(uint32_t value) { #ifdef __GNUC__ return __builtin_ctz(value); #else // __GNUC__ // From https://stackoverflow.com/a/757266 static const int MultiplyDeBruijnBitPosition[32] = {0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9}; return MultiplyDeBruijnBitPosition[((uint32_t)((value & -value) * 0x077CB531U)) >> 27]; #endif // __GNUC__ } static int PopCount(uint32_t value) { #ifdef __GNUC__ return __builtin_popcount(value); #elif defined(_MSC_VER) return __popcnt(value); #else XGRAMMAR_LOG(FATAL) << "PopCount is not supported on this platform"; #endif } int DoFindZeroFrom(int first_block) const { int position = -1; for (int i = first_block; i < buffer_size_; ++i) { if (data_[i] != ~static_cast(0)) { position = i; break; } } if (position == -1) return -1; return position * BITS_PER_BLOCK + LowestBit(~data_[position]); } int DoFindOneFrom(int first_block) const { int position = -1; for (int i = first_block; i < buffer_size_; ++i) { if (data_[i] != 0) { position = i; break; } } if (position == -1) return -1; return position * BITS_PER_BLOCK + LowestBit(data_[position]); } // The size of the bitset. int size_; // The size of the buffer. int buffer_size_; // The buffer for the bitset. uint32_t* data_; // The internal buffer. It is empty if not needed. std::vector internal_buffer_; // Whether the buffer is internally managed. bool is_internal_; }; } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_DYNAMIC_BITSET_H_ xgrammar-0.2.3/cpp/support/encoding.h000066400000000000000000000365571521764210300176100ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/support/encoding.h * \brief Encoding and decoding from/to UTF-8 and escape sequence to/from codepoints. */ #ifndef XGRAMMAR_SUPPORT_ENCODING_H_ #define XGRAMMAR_SUPPORT_ENCODING_H_ // TODO(yixin): enhance performance #include #include #include #include #include #include #include #include "logging.h" namespace xgrammar { /*! \brief Represents a unicode codepoint. */ using TCodepoint = int32_t; /*! * \brief Represents an error when handling characters. Will be returned as a special TCodepoint * value. */ enum CharHandlingError : TCodepoint { /*! \brief The UTF-8 string is invalid. */ kInvalidUTF8 = -10, /*! \brief The escape sequence is invalid. */ kInvalidEscape = -11, /*! \brief The Latin-1 string is invalid. */ kInvalidLatin1 = -12, }; /******************** UTF-8 Handling ********************/ /*! * \brief Print a codepoint to a UTF-8 string. * \param codepoint The codepoint. * \return The UTF-8 string. */ std::string CharToUTF8(TCodepoint codepoint); /*! * \brief Handle the utf-8 first byte. * \returns (is_valid, total_number_of_bytes, initial_codepoint). */ std::tuple HandleUTF8FirstByte(uint8_t byte); /*! * \brief Parse all codepoints in a UTF-8 string. * \param utf8 The UTF-8 string. * \param perserve_invalid_bytes If the invalid UTF8 bytes will be preserved in the result. * \return All codepoints. If the UTF-8 string is invalid, when perserve_invalid_bytes is false, * the invalid bytes will be added to the result as a TCodepoint. Otherwise, the function will * return {CharHandlingError::kInvalidUTF8}. */ std::vector ParseUTF8(const char* utf8, bool perserve_invalid_bytes = false); /*! * \brief Parse the first codepoint in a UTF-8 string. * \param utf8 The UTF-8 string. * \return The codepoint and the number of bytes consumed. If the UTF-8 string is invalid, return * {CharHandlingError::kInvalidUTF8, 0}. */ std::pair ParseNextUTF8(const char* utf8); /*! * \brief Convert a Latin-1 string to a byte sequence. * \param latin1 The Latin-1 string. * \return The byte sequence. */ std::optional Latin1ToBytes(const std::string& latin1, std::string* result); /******************** Escape Handling ********************/ /*! * \brief Convert a codepoint to a escaped string. If the codepoint is not printable, it will be * escaped. By default the function support escape sequences in C ("\n", "\t", "\u0123"). User * can specify more escape sequences using additional_escape_map. * \param codepoint The codepoint. * \param additional_escape_map A map from codepoint to escape sequence. If the codepoint is in * the map, it will be escaped using the corresponding escape sequence. e.g. {{'-', "\\-"}}. * \return The printable string. */ std::string EscapeString( TCodepoint codepoint, const std::unordered_map& additional_escape_map = {} ); /*! * \brief Convert the given char to a escaped string that can be printed. * \return The escaped string. */ std::string EscapeString(uint8_t raw_char); /*! * \brief Convert the given string to a escaped string that can be printed. * \return The escaped string. */ std::string EscapeString(std::string raw_str); /*! * \brief Convert a hex character to an integer. * \param c The hex character: 0-9, a-f, A-F. * \return The integer value of the hex character. If the character is not a valid hex character, * return -1. */ int HexCharToInt(char c); /*! * \brief Parse the first escaped codepoint from a escaped string. data must start with a '\' * character. * \param data The escaped string. Can be TCodepoint* (e.g. string decoded from UTF-8) or char*. * \param additional_escape_map A map from escape sequence to codepoint. If the escape sequence is * in the map, it will be converted to the corresponding codepoint. e.g. {{"\\-", '-'}}. * \return The codepoint and the number of bytes consumed. */ template std::pair ParseNextEscaped( const CharType* data, const std::unordered_map& additional_escape_map = {} ); /*! * \brief Parse the first codepoint from a UTF-8 string. Also checks escape sequences and converts * the escaped char to its original value. * \param utf8 The UTF-8 string or the escape sequence. * \param additional_escape_map A map from escape sequence to codepoint. If the escape sequence is * in the map, it will be converted to the corresponding codepoint. e.g. {{"\\-", '-'}}. * \return The codepoint and the number of bytes consumed. If the UTF-8 string is invalid, the * function returns (CharHandlingError::kInvalidUTF8, 0). If the escape sequence is invalid, the * function returns (CharHandlingError::kInvalidEscape, 0). */ std::pair ParseNextUTF8OrEscaped( const char* utf8, const std::unordered_map& additional_escape_map = {} ); /******************** Implementation ********************/ inline std::string CharToUTF8(TCodepoint codepoint) { XGRAMMAR_DCHECK(codepoint <= 0x10FFFF) << "Invalid codepoint: " << codepoint; std::string utf8; if (codepoint <= 0x7F) { // 1-byte sequence utf8 += static_cast(codepoint); } else if (codepoint <= 0x7FF) { // 2-byte sequence utf8 += static_cast(0xC0 | ((codepoint >> 6) & 0x1F)); utf8 += static_cast(0x80 | (codepoint & 0x3F)); } else if (codepoint <= 0xFFFF) { // 3-byte sequence utf8 += static_cast(0xE0 | ((codepoint >> 12) & 0x0F)); utf8 += static_cast(0x80 | ((codepoint >> 6) & 0x3F)); utf8 += static_cast(0x80 | (codepoint & 0x3F)); } else { // 4-byte sequence utf8 += static_cast(0xF0 | ((codepoint >> 18) & 0x07)); utf8 += static_cast(0x80 | ((codepoint >> 12) & 0x3F)); utf8 += static_cast(0x80 | ((codepoint >> 6) & 0x3F)); utf8 += static_cast(0x80 | (codepoint & 0x3F)); } return utf8; } inline std::tuple HandleUTF8FirstByte(uint8_t byte) { static const std::array kFirstByteMask = {0x00, 0x7F, 0x1F, 0x0F, 0x07}; // clang-format off static const std::array kUtf8Bytes = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, -1, -1, -1, -1, -1, -1, -1, -1, }; // clang-format on auto num_bytes = kUtf8Bytes[static_cast(byte)]; if (num_bytes == -1) { return {false, 0, 0}; } return {true, num_bytes, byte & kFirstByteMask[num_bytes]}; } inline std::pair ParseNextUTF8(const char* utf8) { auto [accepted, num_bytes, res] = HandleUTF8FirstByte(utf8[0]); if (accepted) { for (int i = 1; i < num_bytes; ++i) { if (utf8[i] == 0 || (static_cast(utf8[i]) & 0xC0) != 0x80) { // invalid utf8 accepted = false; break; } res = (res << 6) | (static_cast(utf8[i]) & 0x3F); } } if (!accepted) { // invalid utf8 return {CharHandlingError::kInvalidUTF8, 0}; } return {res, num_bytes}; } inline std::vector ParseUTF8(const char* utf8, bool perserve_invalid_bytes) { std::vector codepoints; while (*utf8 != 0) { auto [codepoint, num_bytes] = ParseNextUTF8(utf8); if (codepoint == CharHandlingError::kInvalidUTF8) { if (perserve_invalid_bytes) { codepoints.push_back(static_cast(static_cast(utf8[0]))); utf8 += 1; continue; } else { return {CharHandlingError::kInvalidUTF8}; } } codepoints.push_back(codepoint); utf8 += num_bytes; } return codepoints; } /*! \brief Convert a Latin-1 string to a byte sequence. \param latin1 The Latin-1 string. \param result The output byte sequence. The function will convert each Latin-1 character to its corresponding byte(s). For characters in the range [0x00, 0x7F], the corresponding byte is the same as the character. Otherwise, the character should be encoded in two bytes in UTF-8: - First byte: 110xxxxx (0xC0 | (char >> 6)) - Second byte: 10xxxxxx (0x80 | (char & 0x3F)) Example: 0xC3 0xBF -> 0xFF 'A' -> 'A' \return std::nullopt if the conversion is successful. Otherwise, return CharHandlingError::kInvalidLatin1 if the Latin-1 string is invalid. */ inline std::optional Latin1ToBytes( const std::string& latin1, std::string* result ) { result->clear(); result->reserve(latin1.size()); const size_t len = latin1.size(); for (size_t i = 0; i < len; ++i) { unsigned char c1 = static_cast(latin1[i]); if (c1 < 0x80) { result->push_back(static_cast(c1)); } else { if (i + 1 >= len) { return CharHandlingError::kInvalidLatin1; } unsigned char c2 = static_cast(latin1[i + 1]); if ((c2 & 0xC0) != 0x80) { return CharHandlingError::kInvalidLatin1; } int code = ((c1 & 0x1F) << 6) | (c2 & 0x3F); if (code < 0x80 || code > 0xFF) { return CharHandlingError::kInvalidLatin1; } result->push_back(static_cast(code)); ++i; } } return std::nullopt; } /*! \brief Convert a byte sequence to a Latin-1 string. \param Bytes The input byte sequence. \param result The output Latin-1 string. The function will convert each byte in the input to a Latin-1 character. For bytes in the range [0x00, 0x7F], the corresponding Latin-1 character is the same as the byte. For bytes in the range [0x80, 0xFF], the corresponding Latin-1 character is represented by two bytes in UTF-8: - First byte: 110xxxxx (0xC0 | (byte >> 6)) - Second byte: 10xxxxxx (0x80 | (byte & 0x3F)) Example: 0xFF -> 0xC3 0xBF 'A' -> 'A' */ inline void ByteToLatin1(const std::string& bytes, std::string* result) { result->clear(); const char* data = bytes.c_str(); for (int current_idx = 0; *(data + current_idx) != '\0'; current_idx++) { const unsigned char& current_char = static_cast(*(data + current_idx)); // Ascii character, directly add to result. if (current_char <= 0x7F) { result->push_back(static_cast(current_char)); continue; } // not Ascii character, convert to Latin-1. unsigned char latin1_first_byte = 0; unsigned char latin1_second_byte = 0; latin1_first_byte = 0xC0 | (current_char >> 6); latin1_second_byte = 0x80 | (current_char & 0x3F); result->push_back(static_cast(latin1_first_byte)); result->push_back(static_cast(latin1_second_byte)); } } inline int HexCharToInt(char c) { if (c >= '0' && c <= '9') { return c - '0'; } else if (c >= 'a' && c <= 'f') { return c - 'a' + 10; } else if (c >= 'A' && c <= 'F') { return c - 'A' + 10; } else { return -1; } } inline std::string EscapeString( TCodepoint codepoint, const std::unordered_map& additional_escape_map ) { static const std::unordered_map kCodepointToEscape = { {'\'', "\\\'"}, {'\"', "\\\""}, {'\?', "\\?"}, {'\\', "\\\\"}, {'\a', "\\a"}, {'\b', "\\b"}, {'\f', "\\f"}, {'\n', "\\n"}, {'\r', "\\r"}, {'\t', "\\t"}, {'\v', "\\v"}, {'\0', "\\0"}, {'\x1B', "\\e"} }; if (auto it = additional_escape_map.find(codepoint); it != additional_escape_map.end()) { return it->second; } if (auto it = kCodepointToEscape.find(codepoint); it != kCodepointToEscape.end()) { return it->second; } if (codepoint >= 0x20 && codepoint <= 0x7E) { return std::string({static_cast(codepoint)}); } // convert codepoint to hex char prefix = codepoint <= 0xFF ? 'x' : codepoint <= 0xFFFF ? 'u' : 'U'; int width = codepoint <= 0xFF ? 2 : codepoint <= 0xFFFF ? 4 : 8; std::stringstream ss; ss << std::setfill('0') << std::setw(width) << std::hex << codepoint; auto hex = ss.str(); return std::string("\\") + prefix + hex; } inline std::string EscapeString(uint8_t raw_char) { return EscapeString(static_cast(raw_char)); } inline std::string EscapeString(std::string raw_str) { std::string res; auto codepoints = ParseUTF8(raw_str.c_str(), true); for (auto c : codepoints) { res += EscapeString(c); } return res; } template std::pair ParseNextEscaped( const CharType* data, const std::unordered_map& additional_escape_map ) { // C escape characters static const std::unordered_map kEscapeToCodepoint = { // clang-format off {'\'', '\''}, {'\"', '\"'}, {'?', '\?'}, {'\\', '\\'}, {'/', '/'}, {'a', '\a'}, {'b', '\b'}, {'f', '\f'}, {'n', '\n'}, {'r', '\r'}, {'t', '\t'}, {'v', '\v'}, {'0', '\0'}, {'e', '\x1B'} // clang-format on }; if (data[0] != '\\') { return {CharHandlingError::kInvalidEscape, 0}; } bool escape_char_in_escape_range = static_cast(static_cast(data[1])) <= 128; if (!escape_char_in_escape_range) { return {CharHandlingError::kInvalidEscape, 0}; } if (auto it = additional_escape_map.find(static_cast(data[1])); it != additional_escape_map.end()) { return {it->second, 2}; } if (auto it = kEscapeToCodepoint.find(static_cast(data[1])); it != kEscapeToCodepoint.end()) { return {it->second, 2}; } if (data[1] == 'x') { // arbitrary length hex int len = 0; TCodepoint codepoint = 0; int32_t digit; while ((digit = HexCharToInt(data[2 + len])) != -1) { codepoint = codepoint * 16 + digit; ++len; } if (len == 0) { return {CharHandlingError::kInvalidEscape, 0}; } return {codepoint, len + 2}; } else if (data[1] == 'u' || data[1] == 'U') { // 4- or 8-digit hex int len = data[1] == 'u' ? 4 : 8; TCodepoint codepoint = 0; for (int i = 0; i < len; ++i) { auto digit = HexCharToInt(data[i + 2]); if (digit == -1) { return {CharHandlingError::kInvalidEscape, 0}; } codepoint = codepoint * 16 + digit; } return {codepoint, len + 2}; } else { return {CharHandlingError::kInvalidEscape, 0}; } } inline std::pair ParseNextUTF8OrEscaped( const char* utf8, const std::unordered_map& additional_escape_map ) { if (utf8[0] != '\\') { return ParseNextUTF8(utf8); } return ParseNextEscaped(utf8, additional_escape_map); } } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_ENCODING_H_ xgrammar-0.2.3/cpp/support/int_set.h000066400000000000000000000064171521764210300174570ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/support/int_set.h * \brief The header for utilities used in grammar-guided generation. */ #ifndef XGRAMMAR_SUPPORT_INT_SET_H_ #define XGRAMMAR_SUPPORT_INT_SET_H_ #include #include #include #include namespace xgrammar { /*! * \brief Let lhs be the union of lhs and rhs. Suppose that both sets are sorted. * \note No additional vectors are allocated, and the time complexity is O(n) */ inline void IntsetUnion(std::vector* lhs, const std::vector& rhs) { int original_lhs_size = lhs->size(); int rhs_size = rhs.size(); lhs->resize(original_lhs_size + rhs_size); auto it_lhs = lhs->rbegin() + rhs_size; auto it_rhs = rhs.rbegin(); auto it_result = lhs->rbegin(); while (it_lhs != lhs->rend() && it_rhs != rhs.rend()) { if (*it_lhs > *it_rhs) { *it_result = *it_lhs; ++it_lhs; } else if (*it_lhs < *it_rhs) { *it_result = *it_rhs; ++it_rhs; } else { *it_result = *it_lhs; ++it_lhs; ++it_rhs; } ++it_result; } while (it_rhs != rhs.rend()) { *it_result = *it_rhs; ++it_result; ++it_rhs; } auto last = std::unique(lhs->begin(), lhs->end()); lhs->erase(last, lhs->end()); } /*! * \brief Let lhs be the intersection of lhs and rhs. Suppose that both sets are sorted. * \note No additional vector is allocated, and the time complexity is O(n). * \note Support the case where lhs is the universal set by setting lhs to {-1}. The result will be * rhs then. */ inline void IntsetIntersection(std::vector* lhs, const std::vector& rhs) { if (lhs->size() == 1 && (*lhs)[0] == -1) { *lhs = rhs; return; } auto it_lhs = lhs->begin(); auto it_rhs = rhs.begin(); auto it_result = lhs->begin(); while (it_lhs != lhs->end() && it_rhs != rhs.end()) { if (*it_lhs < *it_rhs) { ++it_lhs; } else if (*it_lhs > *it_rhs) { ++it_rhs; } else { *it_result = *it_lhs; ++it_lhs; ++it_rhs; ++it_result; } } lhs->erase(it_result, lhs->end()); } /*! * \brief Let lhs = lhs - rhs. Both sets must be sorted. * \note In-place, no additional vector allocated, O(n) time. */ inline void IntsetDifference(std::vector* lhs, const std::vector& rhs) { auto it_lhs = lhs->begin(); auto it_rhs = rhs.begin(); auto it_result = lhs->begin(); while (it_lhs != lhs->end() && it_rhs != rhs.end()) { if (*it_lhs < *it_rhs) { *it_result++ = *it_lhs++; } else if (*it_lhs > *it_rhs) { ++it_rhs; } else { ++it_lhs; ++it_rhs; } } while (it_lhs != lhs->end()) { *it_result++ = *it_lhs++; } lhs->erase(it_result, lhs->end()); } /*! * \brief Compute result = [0, n) - excluded. excluded must be sorted with values in [0, n). * \note O(n) time. */ inline void IntsetComplement( std::vector* result, int32_t n, const std::vector& excluded ) { result->clear(); result->reserve(n - static_cast(excluded.size())); auto it = excluded.begin(); for (int32_t i = 0; i < n; ++i) { if (it != excluded.end() && *it == i) { ++it; } else { result->push_back(i); } } } } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_INT_SET_H_ xgrammar-0.2.3/cpp/support/json_serializer.h000066400000000000000000000554351521764210300212200ustar00rootroot00000000000000/*! * Copyright (c) 2025 by Contributors * \file xgrammar/support/json_serializer.h * \brief A JSON-based serializer. Automatically generates serialization and deserialization logic * from reflection. */ #ifndef XGRAMMAR_SUPPORT_JSON_SERIALIZER_H_ #define XGRAMMAR_SUPPORT_JSON_SERIALIZER_H_ #include #include #include #include #include #include #include #include #include #include "encoding.h" #include "logging.h" #include "reflection.h" #include "utils.h" #include "xgrammar/exception.h" #include "xgrammar/object.h" namespace xgrammar { /******************** Interfaces ********************/ /*! * \brief Manages the version of the serialized object. The version will be added to the serialized * object, and during deserialization, the object's version must match the current serialization * version in xgrammar. */ class SerializeVersion { public: /*! * \brief Returns the current serialization version. */ static std::string_view GetVersion() { return kXGrammarSerializeVersion; } /*! * \brief Adds the version info to the serialized object. */ static void Apply(picojson::object* object); /*! * \brief Checks if the serialized object's version matches the current serialization version. * \return An error if the version does not exist or does not match. */ static std::optional Check(const picojson::object& object); private: /*! * \brief The key of the version info in the serialized object. */ static constexpr const char kXGrammarSerializeVersionKey[] = "__VERSION__"; /*! * \brief The current serialization version. When the serialization result of any object in * XGrammar is changed, this version should be bumped. */ static constexpr const char kXGrammarSerializeVersion[] = "v14"; }; /*! * \brief Serializes a value to a JSON value. * \details It supports STL types, PImpl types, reflection-based types (whose members are defined * through XGRAMMAR_MEMBER_TABLE or XGRAMMAR_MEMBER_ARRAY), and types who have defined a global * SerializeJSONValue function. For reflection-based types, the serialization logic is automatically * generated from the defined members. * \param value The value to be serialized. * \return The serialized JSON value. */ template picojson::value AutoSerializeJSONValue(const T& value); /*! * \brief Serializes a value to a JSON string. * \details It supports STL types, PImpl types, reflection-based types (whose members are defined * through XGRAMMAR_MEMBER_TABLE or XGRAMMAR_MEMBER_ARRAY), and types who have defined a global * SerializeJSONValue function. For reflection-based types, the serialization logic is automatically * generated from the defined members. * \param value The value to be serialized. * \param add_version Whether to add the version info to the serialized object. The addition is * valid only when the serialized result is an object. * \return The serialized JSON string. */ template std::string AutoSerializeJSON(const T& value, bool add_version = false); /*! * \brief Deserializes a value from a JSON value. * \details It supports STL types, PImpl types, reflection-based types (whose members are defined * through XGRAMMAR_MEMBER_TABLE or XGRAMMAR_MEMBER_ARRAY), and types who have defined a global * DeserializeJSONValue function. For reflection-based types, the deserialization logic is * automatically generated from the defined members. * \param result The pointer to the result to be deserialized. * \param value The JSON value to be deserialized. * \param type_name The name of the type to be deserialized. Used for error message. * \return The deserialization error if any. */ template std::optional AutoDeserializeJSONValue( T* result, const picojson::value& value, const std::string& type_name = "" ); /*! * \brief Deserializes a value from a JSON string. * \details It supports STL types, PImpl types, reflection-based types (whose members are defined * through XGRAMMAR_MEMBER_TABLE or XGRAMMAR_MEMBER_ARRAY), and types who have defined a global * DeserializeJSONValue function. For reflection-based types, the deserialization logic is * automatically generated from the defined members. * \param result The pointer to the result to be deserialized. * \param json_string The JSON string to be deserialized. * \param check_version Whether to check the version info in the serialized object. The check is * valid only when the serialized object is an object. * \param type_name The name of the type to be deserialized. Used for error message. * \return The deserialization error if any. */ template std::optional AutoDeserializeJSON( T* result, const std::string& json_string, bool check_version = false, const std::string& type_name = "" ); /*! * \brief Constructs a deserialize error with the given error message and type name. * \param error_message The error message. * \param type_name The name of the type. * \return The constructed runtime error. */ inline SerializationError ConstructDeserializeError( const std::string& error_message, const std::string& type_name ); /******************** Implementations ********************/ inline void SerializeVersion::Apply(picojson::object* object) { XGRAMMAR_DCHECK(object != nullptr); XGRAMMAR_DCHECK(object->find(kXGrammarSerializeVersionKey) == object->end()); (*object)[kXGrammarSerializeVersionKey] = picojson::value(std::string(GetVersion())); } inline std::optional SerializeVersion::Check(const picojson::object& object) { if (object.find(kXGrammarSerializeVersionKey) == object.end()) { return DeserializeVersionError( std::string("Missing version in serialized object: ") + kXGrammarSerializeVersionKey ); } if (object.at(kXGrammarSerializeVersionKey).get() != GetVersion()) { return DeserializeVersionError( std::string("Wrong version in serialized object: Got ") + object.at(kXGrammarSerializeVersionKey).get() + ", expected " + std::string(GetVersion()) ); } return std::nullopt; } /******************** Template Implementations ********************/ namespace detail::json_serializer { template struct has_serialize_json_global : std::false_type {}; template struct has_serialize_json_global< T, std::void_t()))>> : std::true_type { static_assert( std::is_same_v())), picojson::value>, "SerializeJSONValue must be a global function returning picojson::value" ); }; template struct has_deserialize_json_global : std::false_type {}; template struct has_deserialize_json_global< T, std::void_t(), picojson::value{}, std::string{}) )>> : std::true_type { static_assert( std::is_same_v< decltype(DeserializeJSONValue(std::declval(), picojson::value{}, std::string{})), std::optional>, "DeserializeJSONValue must be a global function returning std::optional" ); static_assert( std::is_default_constructible_v, "global deserializer can only apply to a default constructible type" ); }; template inline constexpr bool false_v = false; template inline picojson::value TraitSerializeJSONValue(const T& value) { using Functor = member_functor; if constexpr (Functor::value == member_type::kConfig) { if constexpr (Functor::has_names) { // normal named struct picojson::object obj; obj.reserve(Functor::member_count); visit_config([&](auto ptr, const char* name, std::size_t) { XGRAMMAR_DCHECK(obj.find(name) == obj.end()); obj[name] = AutoSerializeJSONValue(value.*ptr); }); return picojson::value(std::move(obj)); } else if constexpr (Functor::member_count == 1) { // optimize for single member unnamed structs constexpr auto member_ptr = std::get<0>(Functor::members); return AutoSerializeJSONValue(value.*member_ptr); } else { // normal unnamed struct picojson::array arr; arr.resize(Functor::member_count); visit_config([&](auto ptr, const char*, std::size_t idx) { arr[idx] = AutoSerializeJSONValue(value.*ptr); }); return picojson::value(std::move(arr)); } } else { // should give an error in this case static_assert(detail::json_serializer::false_v, "Invalid trait type"); return picojson::value{}; } } template inline std::optional TraitDeserializeJSONValue( T* result, const picojson::value& value, const std::string& type_name ) { using Functor = member_functor; if constexpr (Functor::value == member_type::kConfig) { if constexpr (Functor::has_names) { // normal named struct if (!value.is()) { return ConstructDeserializeError("Expect an object", type_name); } const auto& obj = value.get(); std::optional err = std::nullopt; visit_config([&](auto ptr, const char* name, std::size_t idx) { if (err) { return; } else if (obj.find(name) == obj.end()) { err = ConstructDeserializeError("Missing member " + std::string(name), type_name); } else if (auto e = AutoDeserializeJSONValue(&(result->*ptr), obj.at(name), type_name)) { err = e; } }); return err; } else if constexpr (Functor::member_count == 1) { // optimize for single member unnamed structs constexpr auto member_ptr = std::get<0>(Functor::members); return AutoDeserializeJSONValue(&(result->*member_ptr), value, type_name); } else { // normal unnamed struct if (!value.is()) { return ConstructDeserializeError("Expect an array", type_name); } const auto& arr = value.get(); if (arr.size() != Functor::member_count) { return ConstructDeserializeError( "Wrong number of elements in array: Expected " + std::to_string(Functor::member_count) + ", but got " + std::to_string(arr.size()), type_name ); } std::optional err = std::nullopt; visit_config([&](auto ptr, const char*, std::size_t idx) { if (err) { return; } else if (auto e = AutoDeserializeJSONValue(&(result->*ptr), arr[idx], type_name)) { err = e; } }); return err; } } else { // should give an error in this case static_assert(detail::json_serializer::false_v, "Invalid trait type"); XGRAMMAR_UNREACHABLE(); } } /******************** Customized Serialization ********************/ template > inline picojson::value AutoSerializeJSONValuePImpl(const T& value) { if (value.IsNull()) return picojson::value{}; return AutoSerializeJSONValue(*value.ImplPtr()); } template > inline std::optional AutoDeserializeJSONValuePImpl( T* result, const picojson::value& value, const std::string& type_name ) { XGRAMMAR_DCHECK(result->IsNull()); if (value.is()) { *result = T{NullObj{}}; return std::nullopt; } auto ptr = std::make_shared(); if (auto error = AutoDeserializeJSONValue(ptr.get(), value, type_name)) { return error; } *result = T(std::move(ptr)); return std::nullopt; } } // namespace detail::json_serializer inline SerializationError ConstructDeserializeError( const std::string& error_message, const std::string& type_name ) { if (type_name.empty()) { return DeserializeFormatError("Deserialize error: " + error_message); } else { return DeserializeFormatError("Deserialize error for type " + type_name + ": " + error_message); } } template inline picojson::value AutoSerializeJSONValue(const T& value) { if constexpr (detail::json_serializer::has_serialize_json_global::value) { // User-defined SerializeJSONValue (highest priority) return SerializeJSONValue(value); } else if constexpr (is_pimpl_class::value) { // Library-customized serialization methods return detail::json_serializer::AutoSerializeJSONValuePImpl(value); } else if constexpr (member_trait::value != member_type::kNone) { // Trait serialization methods return detail::json_serializer::TraitSerializeJSONValue(value); } else if constexpr (std::is_same_v) { // Below is primitive types return picojson::value(value); } else if constexpr (std::is_integral_v || std::is_enum_v) { return picojson::value(static_cast(value)); } else if constexpr (std::is_floating_point_v) { return picojson::value(static_cast(value)); } else if constexpr (std::is_same_v) { std::string result; ByteToLatin1(value, &result); return picojson::value(result); } else if constexpr (is_std_optional::value) { if (value.has_value()) { return AutoSerializeJSONValue(*value); } else { return picojson::value{}; } } else if constexpr (is_std_pair::value) { // std::pair: serialize as an array of size 2 picojson::array arr; arr.resize(2); arr[0] = AutoSerializeJSONValue(value.first); arr[1] = AutoSerializeJSONValue(value.second); return picojson::value(std::move(arr)); } else if constexpr (is_std_vector::value) { picojson::array arr; arr.reserve(value.size()); for (const auto& item : value) { arr.push_back(AutoSerializeJSONValue(item)); } return picojson::value(std::move(arr)); } else if constexpr (is_std_unordered_set::value) { std::vector ptr_vec; ptr_vec.reserve(value.size()); for (const auto& item : value) { ptr_vec.push_back(&item); } std::sort(ptr_vec.begin(), ptr_vec.end(), [](const auto* a, const auto* b) { return *a < *b; }); picojson::array arr; arr.reserve(value.size()); for (const auto* ptr : ptr_vec) { arr.push_back(AutoSerializeJSONValue(*ptr)); } return picojson::value(std::move(arr)); } else if constexpr (is_std_unordered_map::value) { if constexpr (std::is_same_v) { // unordered_map: map to json object picojson::object obj; obj.reserve(value.size()); for (const auto& item : value) { obj[item.first] = AutoSerializeJSONValue(item.second); } return picojson::value(std::move(obj)); } else { // unordered_map (T1 is not string): map to json array of array of size 2 std::vector ptr_vec; ptr_vec.reserve(value.size()); for (const auto& item : value) { ptr_vec.push_back(&item); } std::sort(ptr_vec.begin(), ptr_vec.end(), [](const auto* a, const auto* b) { return a->first < b->first; }); picojson::array arr; arr.reserve(value.size()); for (const auto* ptr : ptr_vec) { const auto& [key, item] = *ptr; picojson::array sub_arr{AutoSerializeJSONValue(key), AutoSerializeJSONValue(item)}; arr.push_back(picojson::value(std::move(sub_arr))); } return picojson::value(std::move(arr)); } } else { // should give an error in this case static_assert(detail::json_serializer::false_v, "Cannot serialize this type"); XGRAMMAR_UNREACHABLE(); } } template inline std::string AutoSerializeJSON(const T& value, bool add_version) { picojson::value json_value = AutoSerializeJSONValue(value); if (add_version) { XGRAMMAR_DCHECK(json_value.is()); SerializeVersion::Apply(&json_value.get()); } return picojson::value(json_value).serialize(); } template inline std::optional AutoDeserializeJSONValue( T* result, const picojson::value& value, const std::string& type_name ) { static_assert(!std::is_const_v, "Cannot deserialize into a const type"); if constexpr (detail::json_serializer::has_deserialize_json_global::value) { return DeserializeJSONValue(result, value, type_name); } else if constexpr (is_pimpl_class::value) { return detail::json_serializer::AutoDeserializeJSONValuePImpl(result, value, type_name); } else if constexpr (member_trait::value != member_type::kNone) { return detail::json_serializer::TraitDeserializeJSONValue(result, value, type_name); } else if constexpr (std::is_same_v) { if (!value.is()) { return ConstructDeserializeError("Expect a boolean", type_name); } *result = value.get(); return std::nullopt; } else if constexpr (std::is_integral_v || std::is_enum_v) { if (!value.is()) { return ConstructDeserializeError("Expect an integer", type_name); } *result = static_cast(value.get()); return std::nullopt; } else if constexpr (std::is_floating_point_v) { if (!value.is()) { return ConstructDeserializeError("Expect a floating point number", type_name); } *result = static_cast(value.get()); return std::nullopt; } else if constexpr (std::is_same_v) { if (!value.is()) { return ConstructDeserializeError("Expect a string", type_name); } // Now PicoJSON will convert byte sequence to latin-1 string. Convert it back to byte sequence. auto error = Latin1ToBytes(value.get(), result); if (error) { return ConstructDeserializeError( "XGramamr serializer will serialize byte sequence as latin-1 string, but got invalid " "latin-1 string", type_name ); } return std::nullopt; } else if constexpr (is_std_optional::value) { // for the following container, T must be default constructible if (value.is()) { result->reset(); return std::nullopt; } else { return AutoDeserializeJSONValue(&(result->emplace()), value, type_name); } } else if constexpr (is_std_pair::value) { // std::pair: deserialize from an array of size 2 if (!value.is()) { return ConstructDeserializeError("Expect an array for deserializing pair", type_name); } const auto& arr = value.get(); if (arr.size() != 2) { return ConstructDeserializeError( "Expect an array of size 2 for deserializing pair", type_name ); } if (auto error = AutoDeserializeJSONValue(&(result->first), arr[0], type_name)) { return error; } if (auto error = AutoDeserializeJSONValue(&(result->second), arr[1], type_name)) { return error; } return std::nullopt; } else if constexpr (is_std_vector::value) { if (!value.is()) { return ConstructDeserializeError("Expect an array", type_name); } const auto& arr = value.get(); result->clear(); result->reserve(arr.size()); for (const auto& item : arr) { if (auto error = AutoDeserializeJSONValue(&(result->emplace_back()), item, type_name)) { return error; } } return std::nullopt; } else if constexpr (is_std_unordered_set::value) { if (!value.is()) { return ConstructDeserializeError( "Expect an array for deserializing unordered set", type_name ); } const auto& arr = value.get(); result->clear(); result->reserve(arr.size()); for (const auto& item : arr) { typename T::value_type item_value{}; if (auto error = AutoDeserializeJSONValue(&item_value, item, type_name)) { return error; } result->emplace(std::move(item_value)); } return std::nullopt; } else if constexpr (is_std_unordered_map::value) { if constexpr (std::is_same_v) { // unordered_map: convert from json object if (!value.is()) { return ConstructDeserializeError("Expect an object", type_name); } const auto& obj = value.get(); result->clear(); result->reserve(obj.size()); for (const auto& [key, item] : obj) { typename T::mapped_type item_value{}; if (auto error = AutoDeserializeJSONValue(&item_value, item, type_name)) { return error; } result->try_emplace(key, std::move(item_value)); } return std::nullopt; } else { // unordered_map (T1 is not string): convert from json array of array of size 2 if (!value.is()) { return ConstructDeserializeError( "Expect an array for deserializing unordered map", type_name ); } const auto& arr = value.get(); result->clear(); result->reserve(arr.size()); for (const auto& item : arr) { if (!item.is()) { return ConstructDeserializeError( "Expect an array of array of size 2 for deserializing unordered map", type_name ); } const auto& sub_arr = item.get(); if (sub_arr.size() != 2) { return ConstructDeserializeError( "Expect an array of array of size 2 for deserializing unordered map", type_name ); } typename T::key_type key_value{}; if (auto error = AutoDeserializeJSONValue(&key_value, sub_arr[0], type_name)) { return error; } typename T::mapped_type item_value{}; if (auto error = AutoDeserializeJSONValue(&item_value, sub_arr[1], type_name)) { return error; } result->emplace(std::move(key_value), std::move(item_value)); } return std::nullopt; } } else { // should give an error in this case static_assert(detail::json_serializer::false_v, "Cannot deserialize this type"); XGRAMMAR_UNREACHABLE(); } } template inline std::optional AutoDeserializeJSON( T* result, const std::string& json_string, bool check_version, const std::string& type_name ) { picojson::value json_value; if (auto error = picojson::parse(json_value, json_string); !error.empty()) { return InvalidJSONError(error); } if (check_version) { XGRAMMAR_DCHECK(json_value.is()); if (auto error = SerializeVersion::Check(json_value.get())) { return error; } } return AutoDeserializeJSONValue(result, json_value, type_name); } } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_JSON_SERIALIZER_H_ xgrammar-0.2.3/cpp/support/logging.cc000066400000000000000000000010111521764210300175570ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/support/logging.cc */ #include "logging.h" namespace xgrammar { #if XGRAMMAR_LOG_CUSTOMIZE == 0 LogFatal::Entry& LogFatal::GetEntry() { static thread_local LogFatal::Entry result; return result; } const char* LogMessage::level_strings_[] = { ": ", // XGRAMMAR_LOG_LEVEL_INFO ": Debug: ", // XGRAMMAR_LOG_LEVEL_DEBUG ": Warning: ", // XGRAMMAR_LOG_LEVEL_WARNING }; #endif // XGRAMMAR_LOG_CUSTOMIZE } // namespace xgrammar xgrammar-0.2.3/cpp/support/logging.h000066400000000000000000000161031521764210300174310ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/support/logging.h * \brief A logging library that supports logging at different levels. */ #ifndef XGRAMMAR_SUPPORT_LOGGING_H_ #define XGRAMMAR_SUPPORT_LOGGING_H_ #include #include #include #include #include #include "cpptrace.h" // IWYU pragma: keep /*! * \brief Whether or not customize the logging output. * If log customize is enabled, the user must implement * xgrammar::LogFatalImpl and xgrammar::LogMessageImpl. */ #ifndef XGRAMMAR_LOG_CUSTOMIZE #define XGRAMMAR_LOG_CUSTOMIZE 0 #endif namespace xgrammar { // Provide support for customized logging. #if XGRAMMAR_LOG_CUSTOMIZE /*! * \brief Custom implementations of LogFatal. * * \sa XGRAMMAR_LOG_CUSTOMIZE */ [[noreturn]] void LogFatalImpl(const std::string& file, int lineno, const std::string& message); /*! * \brief Custom implementations of LogMessage. * * \sa XGRAMMAR_LOG_CUSTOMIZE */ void LogMessageImpl(const std::string& file, int lineno, int level, const std::string& message); /*! * \brief Class to accumulate an error message and throw it. Do not use * directly, instead use LOG(FATAL). */ class LogFatal { public: LogFatal(const std::string& file, int lineno) : file_(file), lineno_(lineno) {} #ifdef _MSC_VER #pragma disagnostic push #pragma warning(disable : 4722) #endif [[noreturn]] ~LogFatal() noexcept(false) { LogFatalImpl(file_, lineno_, stream_.str()); } #ifdef _MSC_VER #pragma disagnostic pop #endif std::ostringstream& stream() { return stream_; } private: std::ostringstream stream_; std::string file_; int lineno_; }; /*! * \brief Class to accumulate an log message. Do not use directly, instead use * LOG(INFO), LOG(WARNING), LOG(ERROR). */ class LogMessage { public: LogMessage(const std::string& file, int lineno, int level) : file_(file), lineno_(lineno), level_(level) {} ~LogMessage() { LogMessageImpl(file_, lineno_, level_, stream_.str()); } std::ostringstream& stream() { return stream_; } private: std::string file_; int lineno_; int level_; std::ostringstream stream_; }; #else // if XGRAMMAR_LOG_CUSTOMIZE /*! * \brief Error type for errors from XGRAMMAR_CHECK, XGRAMMAR_ICHECK, and XGRAMMAR_LOG(FATAL). This * error contains a backtrace of where it occurred. */ class LogFatalError : public std::runtime_error { public: /*! \brief Construct an error. Not recommended to use directly. Instead use XGRAMMAR_LOG(FATAL). * * \param file The file where the error occurred. * \param lineno The line number where the error occurred. * \param message The error message to display. * \param time The time at which the error occurred. This should be in local time. */ LogFatalError( const std::string& file, int lineno, const std::string& message, std::time_t time = std::time(nullptr) ) : std::runtime_error(message), file_(file), lineno_(lineno), time_(time) { std::ostringstream s; s << "[" << std::put_time(std::localtime(&time), "%H:%M:%S") << "] " << file << ":" << lineno << ": " << message << "\n"; full_message_ = s.str(); } /*! \return The file in which the error occurred. */ const std::string& file() const { return file_; } /*! \return The time at which this error occurred. */ const std::time_t& time() const { return time_; } /*! \return The line number at which this error occurred. */ int lineno() const { return lineno_; } /*! \return The error message. */ const char* what() const noexcept override { return full_message_.c_str(); } private: std::string file_; int lineno_; std::time_t time_; std::string full_message_; }; /*! * \brief Class to accumulate an error message and throw it. Do not use * directly, instead use XGRAMMAR_LOG(FATAL). * \note The `LogFatal` class is designed to be an empty class to reduce stack size usage. * To play this trick, we use the thread-local storage to store its internal data. */ class LogFatal { public: LogFatal(const std::string& file, int lineno) { GetEntry().Init(file, lineno); } #ifdef _MSC_VER #pragma disagnostic push #pragma warning(disable : 4722) #endif [[noreturn]] ~LogFatal() noexcept(false) { GetEntry().Finalize(); throw; } #ifdef _MSC_VER #pragma disagnostic pop #endif std::ostringstream& stream() { return GetEntry().stream_; } private: struct Entry { void Init(const std::string& file, int lineno) { this->stream_.str(""); this->file_ = file; this->lineno_ = lineno; } [[noreturn]] LogFatalError Finalize() noexcept(false) { LogFatalError error(file_, lineno_, stream_.str()); throw error; } std::ostringstream stream_; std::string file_; int lineno_; }; static Entry& GetEntry(); }; /*! * \brief Class to accumulate an log message. Do not use directly, instead use * XGRAMMAR_LOG(INFO), XGRAMMAR_LOG(WARNING), XGRAMMAR_LOG(ERROR). */ class LogMessage { public: LogMessage(const std::string& file, int lineno, int level) { std::time_t t = std::time(nullptr); stream_ << "[" << std::put_time(std::localtime(&t), "%H:%M:%S") << "] " << file << ":" << lineno << level_strings_[level]; } ~LogMessage() { std::cerr << (stream_.str() + "\n"); } std::ostringstream& stream() { return stream_; } private: std::ostringstream stream_; static const char* level_strings_[]; }; #endif // XGRAMMAR_LOG_CUSTOMIZE #define XGRAMMAR_LOG_LEVEL_INFO 0 #define XGRAMMAR_LOG_LEVEL_DEBUG 1 #define XGRAMMAR_LOG_LEVEL_WARNING 2 #define XGRAMMAR_LOG_LEVEL_FATAL 3 #define XGRAMMAR_LOG_INFO LogMessage(__FILE__, __LINE__, XGRAMMAR_LOG_LEVEL_INFO).stream() #define XGRAMMAR_LOG_DEBUG LogMessage(__FILE__, __LINE__, XGRAMMAR_LOG_LEVEL_DEBUG).stream() #define XGRAMMAR_LOG_WARNING LogMessage(__FILE__, __LINE__, XGRAMMAR_LOG_LEVEL_WARNING).stream() #define XGRAMMAR_LOG_FATAL LogFatal(__FILE__, __LINE__).stream() /*! * \brief Log a message at the given level. * \param level The level of the message. Can be INFO, DEBUG, WARNING, FATAL. */ #define XGRAMMAR_LOG(level) XGRAMMAR_LOG_##level /*! * \brief Check if the condition is true. Used for checking the correctness of user inputs. * \param x The condition to check. */ #define XGRAMMAR_CHECK(x) \ if (!(x)) LogFatal(__FILE__, __LINE__).stream() << "Check failed: (" #x << ") is false: " /*! * \brief Check if the condition is true. Used to guarantee some internal conditions in the code. * \param x The condition to check. */ #define XGRAMMAR_ICHECK(x) \ if (!(x)) LogFatal(__FILE__, __LINE__).stream() << "Internal check failed: (" #x << ") is false: " /*! * \brief Check if the condition is true. Used to guarantee some internal conditions in the code. * \note This check is only enabled in debug mode. In release mode, it will be disabled for * efficiency. This should be used in preference to XGRAMMAR_ICHECK. * \param x The condition to check. */ #if XGRAMMAR_ENABLE_INTERNAL_CHECK #define XGRAMMAR_DCHECK(x) XGRAMMAR_ICHECK(x) #else #define XGRAMMAR_DCHECK(x) \ while (false) XGRAMMAR_ICHECK(x) #endif // XGRAMMAR_ENABLE_DCHECK } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_LOGGING_H_ xgrammar-0.2.3/cpp/support/memory_size.h000066400000000000000000000071551521764210300203540ustar00rootroot00000000000000/*! * Copyright (c) 2025 by Contributors * \file xgrammar/support/memory_size.h * \brief Compute the memory consumption of a container in heap memory. */ #ifndef XGRAMMAR_SUPPORT_MEMORY_SIZE_H_ #define XGRAMMAR_SUPPORT_MEMORY_SIZE_H_ #include #include #include #include #include #include "reflection.h" namespace xgrammar { /******************* MemorySize Procotol *******************/ template inline constexpr std::size_t MemorySize(const T& value); template inline constexpr std::size_t MemorySize(const std::pair& pair); template inline constexpr std::size_t MemorySize(const std::tuple& tpl); template inline constexpr std::size_t MemorySize(const std::optional& optional_value); /******************* MemorySize Implementations *******************/ namespace detail::memory_size { /*! * \brief Get the element type of a container. */ template using ElementType = std::decay_t; /*! * \brief A false value for static_assert. */ template inline constexpr bool false_v = false; } // namespace detail::memory_size /*! * \brief Compute the memory consumption of a value. * \tparam T The type of the value. * \param value The value. * \return The memory consumption in heap memory of the value in bytes. */ template inline constexpr std::size_t MemorySize(const T& value) { if constexpr (is_pimpl_class::value) { // Customized MemorySize return MemorySize(*value.ImplPtr()); } else if constexpr (std::is_trivially_copyable_v) { // Primitive type return 0; } else if constexpr (std::is_trivially_copyable_v>) { // Container of primitive type return sizeof(detail::memory_size::ElementType) * std::size(value); } else if constexpr (!std::is_trivially_copyable_v>) { // Container of non-primitive type: sum up the memory size of all elements std::size_t size = sizeof(detail::memory_size::ElementType) * std::size(value); for (const auto& element : value) { size += MemorySize(element); } return size; } else { static_assert(detail::memory_size::false_v, "MemorySize is not implemented for this type"); } } /*! * \brief Compute the memory consumption of a pair. * \tparam T1 The type of the first element. * \tparam T2 The type of the second element. * \param pair The pair. * \return The memory consumption in heap memory of the pair. */ template inline constexpr std::size_t MemorySize(const std::pair& pair) { return MemorySize(pair.first) + MemorySize(pair.second); } /*! * \brief Compute the memory consumption of a tuple. * \tparam Ts The types of the tuple. * \param tpl The tuple. * \return The memory consumption in heap memory of the tuple. */ template inline constexpr std::size_t MemorySize(const std::tuple& tpl) { return std::apply([](auto&&... elems) { return (MemorySize(elems) + ... + 0); }, tpl); } /*! * \brief Compute the memory consumption in heap memory. This function is specialized for * std::optional. * \tparam Tp The type of the optional. * \param range The optional. * \return The memory consumption in heap memory of the optional. */ template inline constexpr std::size_t MemorySize(const std::optional& optional_value) { return optional_value.has_value() ? MemorySize(*optional_value) : 0; } } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_MEMORY_SIZE_H_ xgrammar-0.2.3/cpp/support/recursion_guard.cc000066400000000000000000000027321521764210300213370ustar00rootroot00000000000000/*! * Copyright (c) 2025 by Contributors * \file xgrammar/support/recursion_guard.cc */ #include "recursion_guard.h" #include #include #include #include #include "logging.h" namespace xgrammar { int RecursionGuard::LoadMaxRecursionDepthFromEnv() { const char* env_value = std::getenv(kMaxRecursionDepthEnvVar); if (env_value == nullptr) { return kDefaultMaxRecursionDepth; } int value = 0; std::string_view sv(env_value); // Convert the string to an integer auto result = std::from_chars(sv.data(), sv.data() + sv.size(), value); // Check if the conversion is successful if (result.ec == std::errc::invalid_argument || result.ec == std::errc::result_out_of_range || result.ptr != sv.data() + sv.size() || value <= 0) { XGRAMMAR_LOG(WARNING) << "Env variable XGRAMMAR_MAX_RECURSION_DEPTH is not a valid " "integer or out of range: '" << env_value << "', using default " << kDefaultMaxRecursionDepth; return kDefaultMaxRecursionDepth; } // Check if the value is too large if (value > kMaxReasonableDepth) { XGRAMMAR_LOG(WARNING) << "Env variable XGRAMMAR_MAX_RECURSION_DEPTH too large: " << value << ", clamping to " << kMaxReasonableDepth; return kMaxReasonableDepth; } return value; } std::atomic RecursionGuard::max_recursion_depth_{LoadMaxRecursionDepthFromEnv()}; } // namespace xgrammar xgrammar-0.2.3/cpp/support/recursion_guard.h000066400000000000000000000076401521764210300212040ustar00rootroot00000000000000/*! * Copyright (c) 2025 by Contributors * \file xgrammar/support/recursion_guard.h * \brief The header for recursion depth guard. */ #ifndef XGRAMMAR_SUPPORT_RECURSION_GUARD_H_ #define XGRAMMAR_SUPPORT_RECURSION_GUARD_H_ #include #include #include #include "logging.h" namespace xgrammar { /*! * \brief Thread-safe recursion guard to prevent stack overflow * * This class provides a RAII-style guard that tracks recursion depth * and prevents excessive recursion that could lead to stack overflow. * It uses atomic operations for thread safety and supports configurable * maximum recursion depth. */ class RecursionGuard { public: /*! * \brief Constructor that increments recursion depth * \param current_recursion_depth Pointer to the current recursion depth counter * \throws Logs fatal error if max recursion depth is exceeded */ explicit RecursionGuard(int* current_recursion_depth) : current_depth_ptr_(current_recursion_depth) { auto error = AddRecursionDepth(current_depth_ptr_); XGRAMMAR_CHECK(error == std::nullopt) << error.value().what(); } /*! * \brief Reset the recursion depth to 0 * \param current_recursion_depth Pointer to the current recursion depth counter */ static void ResetRecursionDepth(int* current_recursion_depth) { XGRAMMAR_DCHECK(current_recursion_depth != nullptr); *current_recursion_depth = 0; } /*! * \brief Destructor that decrements recursion depth */ ~RecursionGuard() { SubtractRecursionDepth(current_depth_ptr_); } /*! * \brief Get the maximum allowed recursion depth * \return Current maximum recursion depth limit */ static int GetMaxRecursionDepth() { return max_recursion_depth_.load(std::memory_order_relaxed); } /*! * \brief Set the maximum allowed recursion depth * \param max_depth New maximum recursion depth limit (must be positive) */ static void SetMaxRecursionDepth(int max_depth) { if (max_depth <= 0 || max_depth > kMaxReasonableDepth) { XGRAMMAR_LOG(FATAL ) << "RecursionGuard: Maximum recursion depth must be positive and less than " << kMaxReasonableDepth << ", got: " << max_depth; } max_recursion_depth_.store(max_depth, std::memory_order_relaxed); } static std::optional AddRecursionDepth(int* current_recursion_depth) { XGRAMMAR_DCHECK(current_recursion_depth != nullptr); int current_depth = ++(*current_recursion_depth); int max_depth = max_recursion_depth_.load(std::memory_order_relaxed); if (current_depth > max_depth) { return std::runtime_error( "RecursionGuard: Maximum recursion depth exceeded. " "Current depth: " + std::to_string(current_depth) + ", Max allowed: " + std::to_string(max_depth) ); } return std::nullopt; } static void SubtractRecursionDepth(int* current_recursion_depth) { XGRAMMAR_DCHECK(current_recursion_depth != nullptr && *current_recursion_depth > 0); --(*current_recursion_depth); } private: /*! * \brief Get the maximum allowed recursion depth from the environment variable. Used to * initialize max_recursion_depth_. * \return Current maximum recursion depth limit */ static int LoadMaxRecursionDepthFromEnv(); /*! * \brief Pointer to the recursion depth counter */ int* current_depth_ptr_; /*! * \brief Thread-safe global configuration */ static std::atomic max_recursion_depth_; /*! * \brief Environment variable name for the maximum recursion depth */ inline constexpr static char kMaxRecursionDepthEnvVar[] = "XGRAMMAR_MAX_RECURSION_DEPTH"; /*! * \brief Default maximum recursion depth */ inline constexpr static int kDefaultMaxRecursionDepth = 10000; /*! * \brief Maximum reasonable recursion depth */ inline constexpr static int kMaxReasonableDepth = 1000000; }; } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_RECURSION_GUARD_H_ xgrammar-0.2.3/cpp/support/reflection.h000066400000000000000000000243751521764210300201470ustar00rootroot00000000000000/*! * Copyright (c) 2025 by Contributors * \file xgrammar/support/reflection.h * \brief The header for compile-time reflection. */ #ifndef XGRAMMAR_SUPPORT_REFLECTION_H_ #define XGRAMMAR_SUPPORT_REFLECTION_H_ #include #include #include #include #include #include #include #include #include namespace xgrammar { /******************** Core Reflection Types ********************/ /*! * \brief The type of the member trait. */ enum class member_type { kNone = 0, // this is default, which has no member trait kConfig = 1, // this is a config with member pointers }; /** * \brief Base trait for member traits. * * \tparam T the type whose members are being reflected * \details Provides a default trait indicating no members. */ template struct member_trait { static constexpr auto value = member_type::kNone; }; /******************** STL and Custom Type Traits ********************/ template struct is_std_array : std::false_type {}; template struct is_std_array> : std::true_type {}; template struct is_std_pair : std::false_type {}; template struct is_std_pair> : std::true_type {}; template struct is_std_tuple : std::false_type {}; template struct is_std_tuple> : std::true_type {}; template struct is_std_optional : std::false_type {}; template struct is_std_optional> : std::true_type {}; template struct is_std_vector : std::false_type {}; template struct is_std_vector> : std::true_type {}; template struct is_std_unordered_map : std::false_type {}; template struct is_std_unordered_map> : std::true_type {}; template struct is_std_unordered_set : std::false_type {}; template struct is_std_unordered_set> : std::true_type {}; /*! * \brief XGrammar specific: Check if a class is a PImpl class. */ template struct is_pimpl_class : std::false_type {}; /*! * \brief XGrammar specific: Check if a class is a PImpl class. It's true iff the class has a * member `Impl` and the class is not the same as the `Impl` type. */ template struct is_pimpl_class< T, std::void_t, void>>> : std::true_type {}; /*! * \brief A helper class to print the value when the condition is false. */ template struct DebugAssert { static_assert(condition); }; /******************** Implementation Details ********************/ namespace detail::reflection { // We cannot use `static_assert(false)` even in unreachable code in `if constexpr`. // See https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2023/p2593r1.html // for more details. // TL;DR: We use the following `false_v` as a workaround. template inline constexpr bool false_v = false; // Note that we don't allow empty tables now (that's uncommon). template inline constexpr auto make_member_table(X, Y second, Args... args) { static_assert(sizeof...(args) % 2 == 0, "member table must be even"); static_assert(std::is_same_v, "first member must be a c-string"); static_assert(std::is_member_pointer_v, "second member must be a member pointer"); if constexpr (sizeof...(args) == 0) { return std::make_tuple(second); } else { return std::tuple_cat(std::make_tuple(second), make_member_table(args...)); } } template inline constexpr auto make_name_table_aux(std::index_sequence, Tuple tuple) { return std::array{std::get(tuple)...}; } template inline constexpr auto make_name_table(Args... args) { constexpr auto N = sizeof...(args); static_assert(N % 2 == 0, "name table must be even"); return make_name_table_aux(std::make_index_sequence{}, std::make_tuple(args...)); } template inline void visit_config_impl(Fn&& fn, std::index_sequence) { // This is a helper function to visit each member of the config. // It uses fold expression to apply the function to each member. static_assert(Ftor::value == member_type::kConfig, "T must be a config type"); static constexpr auto get_name = [](std::size_t idx) { if constexpr (Ftor::has_names) { return Ftor::names[idx]; } else { return ""; } }; return (fn(std::get(Ftor::members), get_name(Idx), Idx), ...); } } // namespace detail::reflection /******************** Member Functors and Visitors ********************/ /*! * \brief A functor that provides access to the members of a config type. * It extracts the members from the `member_trait` specialization for the type `T`. * A valid `member_trait` specialization must meet the following requirements: * - It must have a static member `value` of type `member_type`, * which must be either `kNone` or `kConfig`. */ template ::value> struct member_functor { static_assert(detail::reflection::false_v, "This specialization should never be used"); }; /*! * \brief A specialization of `member_functor` for config types. * A valid `member_trait` specialization for a config type must meet the following: * - It must have a static member `value` of type `member_type::kConfig`. * - It must have a static tuple `members` that contains the member pointers. * - It must have a static array `names` that contains the names of the members. * - The size of `names` must be either 0 or equal to the number of members in `members`. * - In the first case, `names` will be empty. * - In the second case, `names` represent the printed name of each member. */ template struct member_functor { private: using _trait_t = member_trait; using _members_t = std::decay_t; using _names_t = std::decay_t; public: static constexpr auto value = member_type::kConfig; static constexpr auto members = _trait_t::members; static constexpr auto names = _trait_t::names; static constexpr auto member_count = std::tuple_size_v<_members_t>; static constexpr auto has_names = names.size() == member_count; // some static_asserts to check the member list and name list static_assert(is_std_tuple<_members_t>::value, "Member list must be a tuple"); static_assert(is_std_array<_names_t>::value, "Name list must be an array"); static_assert(member_count > 0, "Member list must not be empty"); static_assert( names.size() == member_count || names.size() == 0, "Name list must be empty or have the same size as member list" ); }; /*! * \brief Visit the members of a config type. * \tparam T The type of the config. * \tparam Fn The type of the function to visit the members. * \param fn The function to visit the members. fn's signature should be: * \code{.cpp} * (auto ptr, const char* name, size_t idx) -> void * \endcode * where `ptr` is the pointer to the member, `name` is the name of the member, and `idx` is the * index of the member. */ template inline void visit_config(Fn&& fn) { using Ftor = member_functor; return detail::reflection::visit_config_impl( fn, std::make_index_sequence{} ); } /******************** Registration Macros ********************/ /** * \brief Macros to define member traits for types. * \details These macros are used to define the structural information of types * for serialization and reflection purposes. * * Macros: * - \c XGRAMMAR_MEMBER_TABLE: Defines a type with a table of (name, member pointer) pairs. * - \c XGRAMMAR_MEMBER_ARRAY: Defines a type with an array of member pointers. * * Use the `_TEMPLATE` variants for template types. * * \example * \code{.cpp} * // Example of using XGRAMMAR_MEMBER_TABLE to register (name, member pointer) pairs * struct SimpleClass { * int a; * double b; * }; * XGRAMMAR_MEMBER_TABLE(SimpleClass, "name_a", &SimpleClass::a, "name_b", &SimpleClass::b); * * // Or register members as an array with XGRAMMAR_MEMBER_ARRAY * XGRAMMAR_MEMBER_ARRAY(SimpleClass, &SimpleClass::a, &SimpleClass::b); * * // Example of using XGRAMMAR_MEMBER_ARRAY to register members from a derived class * struct Derived : SimpleClass { * std::string c; * }; * XGRAMMAR_MEMBER_TABLE(Derived, "name_a", &Derived::a, "name_b", &Derived::b, "name_c", * &Derived::c); * * // Example of using XGRAMMAR_MEMBER_ARRAY_TEMPLATE for a template type * // If the default constructor/member is private, you need to declare a friend for member_trait. * template * struct TemplateClass { * private: * T value; * TemplateClass() = default; * friend struct member_trait; * }; * template * XGRAMMAR_MEMBER_ARRAY_TEMPLATE(TemplateClass, &TemplateClass::value); * \endcode */ #define XGRAMMAR_MEMBER_TABLE_TEMPLATE(Type, ...) \ struct member_trait { \ static constexpr auto value = member_type::kConfig; \ static constexpr auto members = detail::reflection::make_member_table(__VA_ARGS__); \ static constexpr auto names = detail::reflection::make_name_table(__VA_ARGS__); \ } #define XGRAMMAR_MEMBER_ARRAY_TEMPLATE(Type, ...) \ struct member_trait { \ static constexpr auto value = member_type::kConfig; \ static constexpr auto members = std::make_tuple(__VA_ARGS__); \ static constexpr auto names = std::array{}; \ } #define XGRAMMAR_MEMBER_TABLE(Type, ...) \ template <> \ XGRAMMAR_MEMBER_TABLE_TEMPLATE(Type, __VA_ARGS__) #define XGRAMMAR_MEMBER_ARRAY(Type, ...) \ template <> \ XGRAMMAR_MEMBER_ARRAY_TEMPLATE(Type, __VA_ARGS__) } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_REFLECTION_H_ xgrammar-0.2.3/cpp/support/thread_pool.h000066400000000000000000000145621521764210300203120ustar00rootroot00000000000000/*! * Copyright (c) 2023 by Contributors * \file xgrammar/support/thread_pool.h * \brief Thread pool. */ #ifndef XGRAMMAR_SUPPORT_THREAD_POOL_H_ #define XGRAMMAR_SUPPORT_THREAD_POOL_H_ #include #include #include #include #include #include #include #include #include "logging.h" namespace xgrammar { /*! * \brief A thread pool implementation for parallel task execution. * * ThreadPool manages a pool of worker threads that can execute tasks asynchronously. * Tasks are submitted to a queue and executed by available threads from the pool. * The pool automatically handles thread synchronization and task distribution. */ class ThreadPool { public: /*! * \brief Construct a new thread pool with the specified number of threads. * \param num_threads Number of worker threads to create. Defaults to hardware concurrency. * \note The pool starts the worker threads immediately upon construction. */ ThreadPool(size_t num_threads = std::thread::hardware_concurrency()) { // Initialize thread pool with num_threads threads for (size_t i = 0; i < num_threads; ++i) { workers_.emplace_back([this] { while (true) { std::function task; { // Lock queue while waiting for new task std::unique_lock lock(queue_mutex_); queue_condition_.wait(lock, [this] { return shutdown_ || !task_queue_.empty(); }); // Exit thread if shutdown and queue is empty if (shutdown_ && task_queue_.empty()) return; // Get task from queue task = std::move(task_queue_.front()); task_queue_.pop(); } task(); TaskComplete(); } }); } } /*! * \brief Add a new task to be executed by the thread pool. * \tparam F Type of the function to execute * \tparam Args Types of the arguments to pass to the function * \param f Function to execute * \param args Arguments to pass to the function * \return std::shared_future containing the result of the function call * \note Tasks are executed in FIFO order but may complete in any order. */ template auto Submit(F&& f, Args&&... args) -> std::shared_future> { using return_type = std::invoke_result_t; // Package the task with its arguments into a shared pointer auto task = std::make_shared>( std::bind(std::forward(f), std::forward(args)...) ); std::shared_future res = task->get_future().share(); { std::unique_lock lock(queue_mutex_); XGRAMMAR_CHECK(!shutdown_) << "Cannot submit task to stopped ThreadPool"; ++unfinished_task_count_; // Increment task count // Directly add the task without wrapping task_queue_.emplace([task]() { (*task)(); }); } queue_condition_.notify_one(); return res; } /*! * \brief Add a new task to be executed by the thread pool without returning a future. * \tparam F Type of the function to execute * \tparam Args Types of the arguments to pass to the function * \param f Function to execute * \param args Arguments to pass to the function * \note Tasks are executed asynchronously by the worker threads. */ template void Execute(F&& f, Args&&... args) { { std::unique_lock lock(queue_mutex_); XGRAMMAR_CHECK(!shutdown_) << "Cannot execute task in stopped ThreadPool"; ++unfinished_task_count_; // Increment task count // Directly add the task without wrapping task_queue_.emplace(std::bind(std::forward(f), std::forward(args)...)); } queue_condition_.notify_one(); } void Wait() { std::unique_lock lock(queue_mutex_); tasks_done_condition_.wait(lock, [this] { return unfinished_task_count_ == 0; }); } /*! * \brief Join all threads in the pool. * * Sets shutdown flag and waits for all threads to complete their current tasks * before destroying the pool. Any remaining tasks in the queue will be executed * before shutdown completes. */ void Join() { { std::unique_lock lock(queue_mutex_); if (shutdown_) return; // Already shut down shutdown_ = true; } queue_condition_.notify_all(); // Wake up all threads so they can exit for (std::thread& worker : workers_) { if (worker.joinable()) worker.join(); // Wait for thread to finish } } /*! * \brief Destructor that ensures graceful shutdown of the thread pool. */ ~ThreadPool() { Join(); } // Prevent copying or moving of the thread pool ThreadPool(const ThreadPool&) = delete; ThreadPool(ThreadPool&&) = delete; ThreadPool& operator=(const ThreadPool&) = delete; ThreadPool& operator=(ThreadPool&&) = delete; private: void TaskComplete() { std::unique_lock lock(queue_mutex_); --unfinished_task_count_; if (unfinished_task_count_ == 0) { tasks_done_condition_.notify_all(); // Notify waiting threads } } /*! \brief Thread container */ std::vector workers_; /*! \brief Task queue */ std::queue> task_queue_; /*! \brief Mutex to protect task queue */ std::mutex queue_mutex_; /*! \brief Condition variable for thread synchronization */ std::condition_variable queue_condition_; /*! \brief Condition variable for task completion */ std::condition_variable tasks_done_condition_; /*! \brief Flag to indicate thread pool shutdown */ bool shutdown_ = false; /*! \brief Number of unfinished tasks */ int unfinished_task_count_ = 0; }; inline void ParallelFor(int low, int high, int num_threads, std::function f) { if (high - low == 1) { f(low); return; } ThreadPool pool(num_threads); int total = high - low; int chunk_size = (total + num_threads - 1) / num_threads; for (int t = 0; t < num_threads; ++t) { int start = low + t * chunk_size; int end = std::min(start + chunk_size, high); if (start >= end) break; // No more iterations to process pool.Execute([f, start, end]() { for (int i = start; i < end; ++i) { f(i); } }); } pool.Join(); } } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_THREAD_POOL_H_ xgrammar-0.2.3/cpp/support/thread_safe_cache.h000066400000000000000000000325761521764210300214070ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/support/thread_safe_cache.h * \brief The header for thread-safe caching functionality. */ #ifndef XGRAMMAR_SUPPORT_THREAD_SAFE_CACHE_H_ #define XGRAMMAR_SUPPORT_THREAD_SAFE_CACHE_H_ #include #include // IWYU pragma: keep #include #include #include #include #include #include #include #include #include "container.h" namespace xgrammar { /*! * \brief Primary template for ThreadSafeCache * \details This class provides thread-safe caching functionality in two forms: * 1. Single value cache when only Value template parameter is provided * 2. Key-value cache when both Key and Value template parameters are provided */ template class ThreadSafeCache; /*! * \brief Thread-safe cache for a single computed value * \tparam Value The type of value being cached * \details Specialization that provides: * - Thread-safe access to a single cached value * - Lazy computation on first access * - Reader-writer locking for concurrent reads */ template class ThreadSafeCache { public: /*! * \brief Constructs a new single-value cache * \param compute The function that computes the cached value */ explicit ThreadSafeCache(std::function compute) : compute_(std::move(compute)) {} /*! * \brief Gets or computes the cached value * \return The cached or newly computed value */ Value Get() { // First try reading from cache with shared lock { std::shared_lock cache_lock(cache_mutex_); if (cache_.has_value()) { return cache_.value(); // Cache hit } } // Acquire exclusive lock to compute value std::unique_lock cache_lock(cache_mutex_); // Double-check to prevent redundant computation if (cache_.has_value()) { return cache_.value(); } Value value = compute_(); XGRAMMAR_DCHECK(!cache_.has_value()); cache_ = value; return value; } /*! * \brief Clears the cached value * This function removes the cached value, so the next call to Get() will recompute it. */ void Clear() { std::unique_lock cache_lock(cache_mutex_); cache_.reset(); } private: /*! \brief Optional container holding the cached value */ std::optional cache_; /*! \brief Function used to compute the value when not cached */ std::function compute_; /*! \brief Reader-writer lock protecting access to cache_ */ std::shared_mutex cache_mutex_; }; /*! * \brief A thread-safe key-value cache with on-demand computation * \tparam Key The type of keys used to lookup values. Should be hashable. * \tparam Value The type of values stored in the cache * \details This cache provides thread-safe access to computed values with the following features: * - Lazy computation: Values are only computed when first requested * - Thread safety: Uses reader-writer locks for concurrent reads * - Parallel computation: Different keys can be computed simultaneously * - Double-checked locking: Prevents redundant computation */ template class ThreadSafeCache { public: /*! * \brief Constructs a new thread-safe cache * \param compute The function that computes values for uncached keys */ explicit ThreadSafeCache(std::function compute) : compute_(std::move(compute)) {} /*! * \brief Gets or computes the value for a key * \param key The key to lookup * \return The cached or newly computed value of the key */ Value Get(const Key& key) { // Why we need this: // - When adding new elements to a unordered_map, the map may be rehashed, // - which means all the iterators may be invalidated. // - However, cppreference says: // - "References and pointers to either key or data stored in the container are only invalidated // - by erasing that element, even when the corresponding iterator is invalidated." // - (See https://en.cppreference.com/w/cpp/container/unordered_map) // - Therefore, we should maintain 2 locks. // - When we add something to the cache, we should hold the cache_mutex_. // - When we erase something from the cache, we should hold the clear_mutex_. auto erase_lock = std::shared_lock(erase_mutex_); // First attempt to read from cache_ { auto cache_lock = std::shared_lock(cache_mutex_); auto it = cache_.find(key); if (it != cache_.end()) { // Cache hit auto& entry = it->second; // The iterator is invalidated after releasing the lock cache_lock.unlock(); // Therefore, we should hold the entry by reference first // We should not hold lock here, since this function may be blocking. return entry.get(compute_, key); } } // Acquire exclusive lock to compute value { auto cache_lock = std::unique_lock(cache_mutex_); auto& entry = cache_[key]; // Create a new entry cache_lock.unlock(); // Release the lock before blocking // We should not hold lock here, since this function may be blocking. return entry.get(compute_, key); } } /*! * \brief Clears all cached values and associated per-key mutexes * This function removes all cached key-value pairs, so subsequent calls to Get() will recompute * them. */ void Clear() { auto erase_lock = std::unique_lock(erase_mutex_); cache_.clear(); } private: struct Entry { Value value; std::once_flag flag; const Value& get(const std::function& f, const Key& key) { // block in this lambda until the value is computed std::call_once(flag, [&] { value = f(key); }); return value; } }; /*! \brief The cache mapping keys to computed values */ std::unordered_map cache_; /*! \brief The function used to compute values for uncached keys */ std::function compute_; /*! \brief Reader-writer lock protecting access to cache_ */ std::shared_mutex cache_mutex_; /*! \brief Mutex protecting removing elements */ std::shared_mutex erase_mutex_; }; namespace details { template class LRUCacheImpl { public: struct Entry { Value value; // value of the node int index; // node index }; /*! \brief Visits the node and moves it to the back of the LRU list. Return its value. */ const Value& LRUVisit(const std::pair& pair) { const auto& entry = pair.second; lru_list_.MoveBack(entry.index); return entry.value; } /*! \brief Initializes the node with the given value and moves it to the back of the LRU list. */ void LRUInit(std::pair& pair, const Value& init) { auto& entry = pair.second; entry.value = init; entry.index = lru_list_.PushBack(&pair).Index(); } /*! * \brief Evicts the least recently used nodes until the predicate returns false. * \param predicate The function that returns true if eviction should continue. * \param evict The function takes a value and returns true if the value can be evicted. * This will be only called when the predicate returns true. * If this function returns true, it should update the size information before return. * \details This function will evict the least recently used nodes until the predicate returns * false. The evict function will be called for each node to determine if it should be evicted. */ template void LRUEvict(const Predicate& predicate, const Evict& evict) { if (!predicate()) return; auto iter = lru_list_.begin(); if (iter == lru_list_.end()) return; do { auto& [key, entry] = **iter; if (evict(entry.value)) { iter = lru_list_.Erase(iter); map_.erase(key); } else { ++iter; // simply skip those waiting for computation } } while (predicate() && iter != lru_list_.end()); } std::unordered_map& GetMap() { return map_; } private: std::unordered_map map_; List*> lru_list_; }; } // namespace details /** * \brief A thread-safe key-value cache with on-demand computation and LRU eviction * \tparam Key The type of keys used to lookup values. Should be hashable. * \tparam Value The type of values stored in the cache * \tparam Computer The functor that computes values for uncached keys * \tparam SizeEstimator The functor that estimates the size of a value in bytes * \details This cache provides thread-safe access to computed values with the following features: * - Lazy computation: Values are only computed when first requested * - LRU eviction: When the cache is full, the least recently used value is evicted * - Thread safety: Uses reader-writer locks for concurrent reads * \attention User should guarantee the following: * 1. The policy class should provide a compute method that takes a key and returns a value. * 2. The value type should have a MemorySize method that returns the size of the value in bytes. */ template class ThreadSafeLRUCache { private: struct SizedValue { Value value; std::size_t size; }; public: inline static constexpr std::size_t kUnlimitedSize = static_cast(-1); explicit ThreadSafeLRUCache( std::size_t max_size = kUnlimitedSize, const Computer& computer = Computer{}, const SizeEstimator& size_estimator = SizeEstimator{} ) : max_size_(max_size), computer_(computer), size_estimator_(size_estimator), cache_() {} std::size_t MaxMemorySize() const { return max_size_; } std::size_t MemorySize() const { return current_size_; } Value Get(const Key& key) { auto future = GetFuture(key); return future.get().value; } void Clear() { // Remove all the ready entries. const auto lock_map = std::lock_guard{map_mutex_}; if (this->max_size_ == kUnlimitedSize) cache_.GetMap().clear(); else cache_.LRUEvict( [] { return true; }, [&](const std::shared_future& value) { // always evict and block until the value is ready try { current_size_ -= value.get().size; } catch (...) { // fine, just ignore the exception, size is not updated } return true; } ); } private: std::shared_future GetFuture(const Key& key) { if (this->max_size_ == kUnlimitedSize) return GetFutureUnlimited(key); auto& map = cache_.GetMap(); { auto lock_map = std::shared_lock{map_mutex_}; auto it = map.find(key); if (it != map.end()) { // We only need to hold LRU lock when shared lock is held here. // When unique lock of map_mutex_ is held, only 1 thread can access the // LRU list at the same time, so we do not need to hold the LRU lock then. const auto lock_lru = std::lock_guard{lru_mutex_}; return cache_.LRUVisit(*it); } } auto task = std::packaged_task{[this, &key] { auto value = computer_(key); auto result = SizedValue{value, size_estimator_(value)}; current_size_ += result.size; return result; }}; auto lock_map = std::unique_lock{map_mutex_}; auto [it, success] = map.try_emplace(key); if (!success) return cache_.LRUVisit(*it); // in this case, we insert the task, and we need to compute the value auto future = task.get_future().share(); // perform eviction if the cache is full cache_.LRUInit(*it, future); cache_.LRUEvict( [&] { return current_size_ > max_size_; }, [&](const std::shared_future& value) { using namespace std::chrono_literals; // if not ready, then do not wait and block here if (value.wait_for(0s) != std::future_status::ready) return false; try { current_size_ -= value.get().size; } catch (...) { // fine, just ignore the exception, size is not updated } return true; } ); // perform the costly computation outside all locks lock_map.unlock(); task(); return future; } std::shared_future GetFutureUnlimited(const Key& key) { auto& map = cache_.GetMap(); { auto lock_map = std::shared_lock{map_mutex_}; auto it = map.find(key); if (it != map.end()) return it->second.value; } auto task = std::packaged_task{[this, &key] { auto value = computer_(key); auto result = SizedValue{value, size_estimator_(value)}; current_size_ += result.size; return result; }}; auto lock_map = std::unique_lock{map_mutex_}; auto [it, success] = map.try_emplace(key); if (!success) return it->second.value; auto future = task.get_future().share(); it->second.value = future; // perform the costly computation outside all locks lock_map.unlock(); task(); return future; } private: const std::size_t max_size_; const Computer computer_; const SizeEstimator size_estimator_; details::LRUCacheImpl> cache_; std::atomic_size_t current_size_{0}; std::shared_mutex map_mutex_; std::mutex lru_mutex_; }; } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_THREAD_SAFE_CACHE_H_ xgrammar-0.2.3/cpp/support/union_find_set.h000066400000000000000000000065641521764210300210200ustar00rootroot00000000000000/*! * Copyright (c) 2025 by Contributors * \file xgrammar/support/union_find_set.h */ #ifndef XGRAMMAR_SUPPORT_UNION_FIND_SET_H_ #define XGRAMMAR_SUPPORT_UNION_FIND_SET_H_ #include #include #include #include #include "logging.h" namespace xgrammar { template class UnionFindSet { private: std::unordered_map> element_to_parent_and_size_; public: UnionFindSet() = default; /*! * \brief Add a new element to the union-find set. * \param element The element to add. * \return True if the element was added successfully, false if it already exists. */ bool Add(const T& element) { if (element_to_parent_and_size_.find(element) != element_to_parent_and_size_.end()) { return false; // Element already exists. } element_to_parent_and_size_[element] = {element, 1}; return true; } /*! \brief Clear the union find set.*/ void Clear() { element_to_parent_and_size_.clear(); } /*! * \brief Find the representative of the set containing the element. * \param element The element to find. * \return The representative of the set containing the element. */ T Find(const T& element) { XGRAMMAR_CHECK(element_to_parent_and_size_.find(element) != element_to_parent_and_size_.end()) << "Element not found in union-find set."; if (element_to_parent_and_size_[element].first != element) { // Path compression. element_to_parent_and_size_[element].first = Find(element_to_parent_and_size_[element].first); } return element_to_parent_and_size_[element].first; } /*! * \brief Union two elements into the same set. * \param a The first element. * \param b The second element. */ void Union(const T& a, const T& b) { XGRAMMAR_CHECK(element_to_parent_and_size_.find(a) != element_to_parent_and_size_.end()) << "Element " << a << " not found in union-find set."; XGRAMMAR_CHECK(element_to_parent_and_size_.find(b) != element_to_parent_and_size_.end()) << "Element " << b << " not found in union-find set."; T root_a = Find(a); T root_b = Find(b); if (root_a == root_b) { return; } if (element_to_parent_and_size_[root_a].second < element_to_parent_and_size_[root_b].second) { std::swap(root_a, root_b); // Make sure root_a is the larger set. } element_to_parent_and_size_[root_b].first = root_a; element_to_parent_and_size_[root_a].second += element_to_parent_and_size_[root_b].second; } int Count(const T& element) const { return element_to_parent_and_size_.count(element); } std::vector> GetAllSets() { std::vector> result; std::unordered_map root_to_set; for (const auto& [value, _] : element_to_parent_and_size_) { auto root = Find(value); if (root_to_set.find(root) == root_to_set.end()) { result.emplace_back(); root_to_set[root] = result.size() - 1; } result[root_to_set[root]].push_back(value); } // Sort result to make it deterministic for (auto& vec : result) { std::sort(vec.begin(), vec.end()); } std::sort(result.begin(), result.end(), [](const std::vector& v1, const std::vector& v2) { return v1.front() < v2.front(); }); return result; } }; } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_UNION_FIND_SET_H_ xgrammar-0.2.3/cpp/support/utils.h000066400000000000000000000351351521764210300171510ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/support/utils.h * \brief Utility functions. */ #ifndef XGRAMMAR_SUPPORT_UTILS_H_ #define XGRAMMAR_SUPPORT_UTILS_H_ #include #include #include #include #include #include #include #include #include "logging.h" /****************** Hash Library ******************/ namespace xgrammar { /*! * \brief Hash and combine value into seed. * \ref https://www.boost.org/doc/libs/1_84_0/boost/intrusive/detail/hash_combine.hpp */ inline void HashCombineBinary(uint64_t& seed, uint64_t value) { seed ^= value + 0x9e3779b97f4a7c15ull + (seed << 6) + (seed >> 2); } /*! * \brief Find the hash sum of several size_t args. */ template inline uint64_t HashCombine(Args... args) { uint64_t seed = 0; (..., HashCombineBinary(seed, args)); return seed; } /*! * \brief Helper class to define the hash function for a struct by its members. */ template struct HashByMembers { std::size_t operator()(T const& x) const noexcept { return HashCombine(std::hash>{}(x.*Members)...); } }; } // namespace xgrammar /*! * \brief Define a hash function for a struct by its members in namespace std. Should be used * outside of namespace xgrammar. * \param Type The type of the struct. * \param ... The member pointers of the struct. * \example * \code * // In the global namespace * XGRAMMAR_HASH_BY_MEMBERS(Type, &Type::member1, &Type::member2, &Type::member3); * \endcode */ #define XGRAMMAR_HASH_BY_MEMBERS(Type, ...) \ namespace std { \ template <> \ struct hash : public xgrammar::HashByMembers {}; \ } /*! * \brief Empty specialization of XGRAMMAR_HASH_BY_MEMBERS. */ #define XGRAMMAR_HASH_BY_MEMBERS_EMPTY(Type) \ namespace std { \ template <> \ struct hash : public xgrammar::HashByMembers {}; \ } namespace std { /*! * \brief Define the hash function for std::pair. */ template struct hash> { size_t operator()(const std::pair& pair) const noexcept { return xgrammar::HashCombine(std::hash{}(pair.first), std::hash{}(pair.second)); } }; /*! * \brief Define the hash function for std::tuple. */ template struct hash> { size_t operator()(const std::tuple& tuple) const noexcept { return std::apply( [](const Args&... args) { return xgrammar::HashCombine(std::hash{}(args)...); }, tuple ); } }; /*! * \brief Define the hash function for std::vector. */ template struct hash> { size_t operator()(const std::vector& vec) const { uint32_t seed = 0; for (const auto& item : vec) { xgrammar::HashCombineBinary(seed, std::hash{}(item)); } return seed; } }; } // namespace std namespace xgrammar { /****************** Result Library ******************/ /*! * \brief A partial result type that can be used to construct a Result. Holds a result value or an * error value. * \tparam T The type of the value * \tparam IsOk Whether the result is ok */ template struct PartialResult { template PartialResult(Args&&... args) : value(std::forward(args)...) {} T value; }; /*! * \brief Construct a success result with the arguments to construct a T. * \tparam T The type of the success value * \tparam Args The types of the arguments to construct a T * \param args The arguments to construct a T * \return A PartialResult with the arguments to construct a T * \example * \code * // Call the constructor of T with the arguments * return ResultOk(1, 2, 3); * \endcode */ template inline PartialResult ResultOk(Args&&... args) { return PartialResult{std::forward(args)...}; } /*! * \brief Construct a success result with a universal reference (both lvalue and rvalue) * \tparam T The type of the success value * \param value The universal reference to the success value * \return A PartialResult with the universal reference to the success value * \example * \code * T value = T(1, 2, 3); * // Move the value to the PartialResult * return ResultOk(std::move(value)); * \endcode */ template inline PartialResult ResultOk(T&& value) { return PartialResult{std::forward(value)}; } /*! * \brief Construct a error result with the arguments to construct a E. * \tparam E The type of the error value. Default to std::runtime_error. * \tparam Args The types of the arguments to construct a E * \param args The arguments to construct a E * \return A PartialResult with the arguments to construct a E * \example * \code * // Construct a std::runtime_error with a error * std::runtime_error error("Message"); * return ResultErr(std::move(error)); * \endcode * \code * // Construct a std::runtime_error with its argument * return ResultErr("Error"); * \endcode * \code * // Construct an E error with its argument * return ResultErr("Error"); * \endcode */ template inline PartialResult ResultErr(Args&&... args) { return PartialResult{std::forward(args)...}; } /*! * \brief Construct a error result with a universal reference (both lvalue and rvalue) * \tparam E The type of the error value * \param err The universal reference to the error value * \return A PartialResult with the universal reference to the error value * \example * \code * E err = E("Error"); * // Move the err to the PartialResult * return ResultErr(std::move(err)); * \endcode */ template inline PartialResult ResultErr(E&& err) { return PartialResult{std::forward(err)}; } /*! * \brief An always-move Result type similar to Rust's Result, representing either success (Ok) or * failure (Err). It always uses move semantics for the success and error values. * \tparam T The type of the success value * \tparam E The type of the error value * * \note The Ok and Err constructor, and all methods of this class (except for ValueRef and ErrRef) * accept only rvalue references as parameters for performance reasons. You should use std::move to * convert a Result to an rvalue reference before invoking these methods. Examples for move * semantics are shown below. * * \example Construct a success result with a rvalue reference * \code * T value; * return Result::Ok(std::move(value)); * \endcode * \example Construct a error result with a rvalue reference of std::runtime_error * \code * std::runtime_error error_msg = std::runtime_error("Error"); * return Result::Err(std::move(error_msg)); * \endcode * \example Construct a error result with a std::runtime_error object constructed with a string * \code * std::string error_msg = "Error"; * return Result::Err(std::move(error_msg)); * \endcode * \example Unwrap the rvalue reference of the result * \code * Result result = func(); * if (result.IsOk()) { * T result_val = std::move(result).Unwrap(); * } else { * std::runtime_error error_msg = std::move(result).UnwrapErr(); * } * \endcode */ template class Result { private: static_assert(!std::is_same_v, "T and E cannot be the same type"); public: /*! \brief Default constructor is deleted to avoid accidental use */ Result() = delete; /*! \brief Construct from Result::Ok */ template >>> Result(PartialResult&& partial_result) : data_(std::in_place_type, std::forward(partial_result.value)) {} /*! \brief Construct from Result::Err */ template >>> Result(PartialResult&& partial_result) : data_(std::in_place_type, std::forward(partial_result.value)) {} /*! \brief Check if Result contains success value */ bool IsOk() const { return std::holds_alternative(data_); } /*! \brief Check if Result contains error */ bool IsErr() const { return std::holds_alternative(data_); } /*! \brief Get the success value. It assumes (or checks if in debug mode) the result is ok. */ T Unwrap() && { XGRAMMAR_DCHECK(IsOk()) << "Called Unwrap() on an Err value"; return std::get(std::move(data_)); } /*! \brief Get the error value. It assumes (or checks if in debug mode) the result is an error. */ E UnwrapErr() && { XGRAMMAR_DCHECK(IsErr()) << "Called UnwrapErr() on an Ok value"; return std::get(std::move(data_)); } /*! \brief Get the success value if present, otherwise return the provided default */ T UnwrapOr(T default_value) && { return IsOk() ? std::get(std::move(data_)) : std::move(default_value); } /*! \brief Map success value to new type using provided function */ template >> Result Map(F&& f) && { if (IsOk()) { return ResultOk(f(std::get(std::move(data_)))); } return ResultErr(std::get(std::move(data_))); } /*! \brief Map error value to new type using provided function */ template >> Result MapErr(F&& f) && { if (IsErr()) { return ResultErr(f(std::get(std::move(data_)))); } return ResultOk(std::get(std::move(data_))); } /*! * \brief Convert a Result to a Result. U should be convertible to T, and V should be * convertible to E. */ template static Result Convert(Result&& result) { if (result.IsOk()) { return ResultOk(std::move(result).Unwrap()); } return ResultErr(std::move(result).UnwrapErr()); } /*! \brief Get a std::variant from the result. */ std::variant ToVariant() && { return std::move(data_); } /*! * \brief Get a reference to the success value. It assumes (or checks if in debug mode) the * result is ok. */ T& ValueRef() & { XGRAMMAR_DCHECK(IsOk()) << "Called ValueRef() on an Err value"; return std::get(data_); } /*! * \brief Get a reference to the error value. It assumes (or checks if in debug mode) the * result is an error. */ E& ErrRef() & { XGRAMMAR_DCHECK(IsErr()) << "Called ErrRef() on an Ok value"; return std::get(data_); } private: // in-place construct T in variant template explicit Result(std::in_place_type_t, Args&&... args) : data_(std::in_place_type, std::forward(args)...) {} // in-place construct E in variant template explicit Result(std::in_place_type_t, Args&&... args) : data_(std::in_place_type, std::forward(args)...) {} std::variant data_; }; /****************** Misc ******************/ // Sometimes GCC fails to detect some branches will not return, such as when we use LOG(FATAL) // to raise an error. This macro manually mark them as unreachable to avoid warnings. #ifdef __GNUC__ #define XGRAMMAR_UNREACHABLE() __builtin_unreachable() #else #define XGRAMMAR_UNREACHABLE() #endif /*! * \brief An error class that contains a type. The type can be an enum. */ template class TypedError : public std::runtime_error { public: explicit TypedError(T type, const std::string& msg) : std::runtime_error(msg), type_(type) {} const T& Type() const noexcept { return type_; } private: T type_; }; /** * \brief Helper function to compare two objects by their members. */ template constexpr bool EqualByMembers(const T& lhs, const T& rhs) noexcept { return std::tie(lhs.*Ms...) == std::tie(rhs.*Ms...); } /** * \brief Define == and != operator for a struct by its members. * \param Type The type of the struct. Must be under namespace xgrammar. * \param ... The member pointers of the struct. * \example * \code * struct Type { * int member1; * std::string member2; * double member3; * * XGRAMMAR_EQUAL_BY_MEMBERS(Type, &Type::member1, &Type::member2, &Type::member3); * }; * \endcode */ #define XGRAMMAR_EQUAL_BY_MEMBERS(Type, ...) \ friend bool operator==(const Type& lhs, const Type& rhs) noexcept { \ return EqualByMembers(lhs, rhs); \ } \ friend bool operator!=(const Type& lhs, const Type& rhs) noexcept { return !(lhs == rhs); } /*! * \brief Empty specialization of XGRAMMAR_EQUAL_BY_MEMBERS. */ #define XGRAMMAR_EQUAL_BY_MEMBERS_EMPTY(Type) \ friend bool operator==(const Type& lhs, const Type& rhs) noexcept { return true; } \ friend bool operator!=(const Type& lhs, const Type& rhs) noexcept { return false; } /*! * \brief Throw an error from a variant of multiple error types. * \param error_variant The variant of multiple error types. * \tparam Args The types of the error types. Each type should inherit from std::runtime_error. */ template [[noreturn]] void ThrowVariantError(const std::variant& error_variant) { std::visit([](const auto& e) { throw e; }, error_variant); XGRAMMAR_UNREACHABLE(); } /*! * \brief Get the message from a variant of multiple error types. * \param error_variant The variant of multiple error types. * \return The message from the error variant. * \tparam Args The types of the error types. Each type should inherit from std::runtime_error. */ template std::string GetMessageFromVariantError(const std::variant& error_variant) { return std::visit([](const auto& e) { return e.what(); }, error_variant); } /*! * \brief Get the type name from a variant of XGrammarError types (each has GetType()). * \param error_variant The variant of multiple error types. * \return The type string from the error variant (e.g. "DeserializeVersionError"). * \tparam Args The types of the error types. Each type should have GetType() const. */ template std::string GetTypeFromVariantError(const std::variant& error_variant) { return std::visit([](const auto& e) { return e.GetType(); }, error_variant); } } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_UTILS_H_ xgrammar-0.2.3/cpp/testing.cc000066400000000000000000000030431521764210300161010ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/testing.cc */ #include "testing.h" #include #include #include #include #include #include #include "grammar_impl.h" #include "grammar_parser.h" #include "support/encoding.h" namespace xgrammar { std::string PrintTokenByIds( const std::vector& token_ids, const TokenizerInfo& tokenizer_info, int max_print_num ) { std::stringstream ss; const auto& sorted_decoded_vocab = tokenizer_info.GetDecodedVocab(); ss << "["; int print_num = std::min(static_cast(token_ids.size()), max_print_num); for (int i = 0; i < print_num; ++i) { ss << "#" << token_ids[i] << " <" << EscapeString(sorted_decoded_vocab[token_ids[i]]) << ">"; if (i < print_num - 1) { ss << ", "; } } if (static_cast(token_ids.size()) > max_print_num) { ss << ", ..."; } ss << "]"; return ss.str(); } Grammar _EBNFToGrammarNoNormalization( const std::string& ebnf_string, const std::string& root_rule_name ) { return ParseEBNF(ebnf_string, root_rule_name); } std::string _PrintGrammarFSMs(const Grammar& grammar) { std::string result; for (int i = 0; i < grammar->NumRules(); i++) { result += "Rule " + std::to_string(i) + ": " + grammar->GetRule(i).name + ", FSM: "; if (grammar->per_rule_fsms[i].has_value()) { result += grammar->per_rule_fsms[i]->GetFsm().ToString(); } else { result += "None"; } result += "\n"; } return result; } } // namespace xgrammar xgrammar-0.2.3/cpp/testing.h000066400000000000000000000012321521764210300157410ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/testing.h * \brief The header testing utilities. */ #ifndef XGRAMMAR_TESTING_H_ #define XGRAMMAR_TESTING_H_ #include #include #include #include #include namespace xgrammar { std::string PrintTokenByIds( const std::vector& token_ids, const TokenizerInfo& tokenizer_info, int max_print_num ); Grammar _EBNFToGrammarNoNormalization( const std::string& ebnf_string, const std::string& root_rule_name ); std::string _PrintGrammarFSMs(const Grammar& grammar); } // namespace xgrammar #endif // XGRAMMAR_TESTING_H_ xgrammar-0.2.3/cpp/tokenizer_info.cc000066400000000000000000000500621521764210300174540ustar00rootroot00000000000000/*! * Copyright (c) 2023 by Contributors * \file xgrammar/tokenizer_info.cc */ #include "xgrammar/tokenizer_info.h" #include #include #include #include #include #include #include #include #include #include "support/encoding.h" #include "support/json_serializer.h" #include "support/logging.h" #include "tokenizer_info_impl.h" #include "xgrammar/exception.h" namespace xgrammar { /************* Token decoders: ByteFallback and ByteLevel *************/ class TokenDecoder { public: /*! * \brief Post-process a raw token to the actual token with the given post-processing method. */ static std::string DecodeToken(const std::string& token, VocabType vocab_type) { // TODO(yixin): Avoid allocating new string in decoder calls if (vocab_type == VocabType::BYTE_FALLBACK) { return SpaceReplacerDecoder(ByteFallbackDecoder(token)); } else if (vocab_type == VocabType::BYTE_LEVEL) { return ByteLevelDecoder(token); } else { return token; } } private: /*! \brief ByteFallback decoder: transform tokens like <0x1B> to hex char byte 1B */ static std::string ByteFallbackDecoder(const std::string& token) { if (token.length() == 6 && token.substr(0, 3) == "<0x" && token.back() == '>') { int byte = 0; for (int i = 0; i < 2; ++i) { byte *= 16; byte += token[3 + i] >= '0' && token[3 + i] <= '9' ? token[3 + i] - '0' : token[3 + i] - 'A' + 10; } XGRAMMAR_CHECK(byte >= 0 && byte < 256); return std::string(/*n=*/1, static_cast(byte)); } return token; } /*! \brief SpaceReplacer decoder: transform "\u2581" back to space */ static std::string SpaceReplacerDecoder(const std::string& token) { // \u2581 is the unicode for "lower one eighth block" // UTF8 encoding for \u2581 is 0xE2 0x96 0x81 std::string result; for (int i = 0; i < static_cast(token.size()); ++i) { if (i + 2 < static_cast(token.size()) && token[i] == char(0xE2) && token[i + 1] == char(0x96) && token[i + 2] == char(0x81)) { result += ' '; i += 2; } else { result += token[i]; } } return result; } /*! * \brief ByteLevel decoder: inverses the bytes-to-unicode transformation in the encoding * process as in * https://github.com/huggingface/transformers/blob/87be06ca77166e6a6215eee5a990ab9f07238a18/src/transformers/models/gpt2/tokenization_gpt2.py#L38-L59 */ static std::string ByteLevelDecoder(const std::string& token) { // The inverse map of bytes_to_unicode. -1 means there is no mapping to this unicode. static const std::array char_to_byte_map = { // clang-format off -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, -1, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 173 // clang-format on }; auto unicode_codepoints = ParseUTF8(token.c_str(), false); if (unicode_codepoints.size() == 1 && unicode_codepoints[0] == kInvalidUTF8) { return token; } std::string decoded; decoded.reserve(unicode_codepoints.size()); for (auto unicode_codepoint : unicode_codepoints) { XGRAMMAR_CHECK(unicode_codepoint >= 0); if (unicode_codepoint >= static_cast(char_to_byte_map.size()) || char_to_byte_map[unicode_codepoint] == -1) { // If there is no mapping, return the original token return token; } decoded += static_cast(char_to_byte_map[unicode_codepoint]); } return decoded; } }; /************* Metadata detection from huggingface tokenizer.json *************/ class HFTokenizerAnalyzer { public: /*! * \brief Detect the vocabulary type from tokenizer.json. * \details Find {"type": "ByteFallback"} or {"type": "ByteLevel"} in "decoder" field of the * tokenizer. */ static VocabType DetectVocabType(const picojson::object& hf_tokenizer_obj) { #define CHECK_AND_WARNING(condition, message) \ if (!(condition)) { \ XGRAMMAR_LOG(WARNING) << "Vocab type detection failed: (" #condition \ << ") is false: " << (message) << " Using RAW VocabType by default."; \ return VocabType::RAW; \ } CHECK_AND_WARNING( hf_tokenizer_obj.count("decoder") && hf_tokenizer_obj.at("decoder").is(), "Decoder field is not found in tokenizer.json." ); auto decoder_obj = hf_tokenizer_obj.at("decoder").get(); CHECK_AND_WARNING( decoder_obj.count("type") && decoder_obj.at("type").is(), "Type field is not found in decoder field" ); auto type = decoder_obj.at("type").get(); std::vector decoders; if (type == "Sequence") { CHECK_AND_WARNING( decoder_obj.count("decoders") && decoder_obj.at("decoders").is(), "Decoders field is not found in a Sequence decoder" ); decoders = decoder_obj.at("decoders").get(); } else { decoders.emplace_back(hf_tokenizer_obj.at("decoder")); } for (const auto& decoder : decoders) { CHECK_AND_WARNING(decoder.is(), "Decoder is not an object"); auto decoder_obj = decoder.get(); CHECK_AND_WARNING( decoder_obj.count("type") && decoder_obj.at("type").is(), "Type field is not found in decoder field" ); auto type = decoder_obj.at("type").get(); if (type == "ByteLevel") { return VocabType::BYTE_LEVEL; } else if (type == "ByteFallback") { return VocabType::BYTE_FALLBACK; } } // If neither byte_level nor byte_fallback decoder is detected, return RAW. return VocabType::RAW; #undef CHECK_AND_WARNING } static bool DetectPrependNormalizer(const picojson::object& hf_tokenizer_obj) { if (!hf_tokenizer_obj.count("normalizer") || !hf_tokenizer_obj.at("normalizer").is()) { return false; } const picojson::value& normalizer_value = hf_tokenizer_obj.at("normalizer"); if (!normalizer_value.is()) { return false; } const picojson::object& normalizer_obj = normalizer_value.get(); if (!normalizer_obj.count("type") || !normalizer_obj.at("type").is()) { return false; } auto type = normalizer_obj.at("type").get(); std::vector normalizers; if (type == "Sequence") { if (!normalizer_obj.count("normalizers") || !normalizer_obj.at("normalizers").is()) { return false; } normalizers = normalizer_obj.at("normalizers").get(); } else { normalizers.emplace_back(normalizer_value); } for (const auto& normalizer : normalizers) { if (!normalizer.is()) { continue; } auto normalizer_obj = normalizer.get(); if (!normalizer_obj.count("type") || !normalizer_obj.at("type").is()) { continue; } auto type = normalizer_obj.at("type").get(); if (type == "Prepend" && normalizer_obj.count("prepend") && normalizer_obj.at("prepend").is() && normalizer_obj.at("prepend").get() == "▁") { return true; } } return false; } static bool DetectMetaspacePreTokenizer(const picojson::object& hf_tokenizer_obj) { if (!hf_tokenizer_obj.count("pre_tokenizer") || !hf_tokenizer_obj.at("pre_tokenizer").is()) { return false; } auto pre_tokenizer_obj = hf_tokenizer_obj.at("pre_tokenizer").get(); if (!pre_tokenizer_obj.count("type") || !pre_tokenizer_obj.at("type").is()) { return false; } auto type = pre_tokenizer_obj.at("type").get(); if (!pre_tokenizer_obj.count("prepend_scheme") || !pre_tokenizer_obj.at("prepend_scheme").is()) { return false; } auto prepend_scheme = pre_tokenizer_obj.at("prepend_scheme").get(); return type == "Metaspace" && (prepend_scheme == "always" || prepend_scheme == "first"); } /*! * \brief Detect whether add prefix space from tokenizer.json. * \details Find {"type": "Prepend", "prepend": "▁"} in "normalizer" field of the tokenizer, or * "pre_tokenizer": {"type": "Metaspace", "prepend_scheme": "always" | "first"} in the tokenizer. */ static bool DetectAddPrefixSpace(const picojson::object& hf_tokenizer_obj) { return DetectPrependNormalizer(hf_tokenizer_obj) || DetectMetaspacePreTokenizer(hf_tokenizer_obj); } }; /************* TokenizerInfo::Impl *************/ bool TokenizerInfo::Impl::IsSpecialToken(const std::string& token) { return token == ""; } TokenizerInfo::Impl::Impl( const std::vector& encoded_vocab, VocabType vocab_type, std::optional vocab_size, std::optional> stop_token_ids, bool add_prefix_space ) : vocab_type_(vocab_type), vocab_size_(vocab_size.value_or(encoded_vocab.size())), add_prefix_space_(add_prefix_space) { decoded_vocab_.reserve(encoded_vocab.size()); sorted_decoded_vocab_.reserve(encoded_vocab.size()); for (int i = 0; i < static_cast(encoded_vocab.size()); ++i) { const std::string& token = TokenDecoder::DecodeToken(encoded_vocab[i], vocab_type_); decoded_vocab_.push_back(token); if ((!stop_token_ids && DETECTION_STOP_TOKENS.count(token)) || (stop_token_ids && std::find(stop_token_ids->begin(), stop_token_ids->end(), i) != stop_token_ids->end())) { stop_token_ids_.push_back(i); } else if (IsSpecialToken(token)) { special_token_ids_.push_back(i); } else { sorted_decoded_vocab_.push_back({i, token}); } } for (int i = encoded_vocab.size(); i < vocab_size_; ++i) { special_token_ids_.push_back(i); } auto f_compare_token = [](const std::pair& a, const std::pair& b) { return a.second < b.second; }; std::sort(sorted_decoded_vocab_.begin(), sorted_decoded_vocab_.end(), f_compare_token); token_id_to_sorted_vocab_index_.assign(vocab_size_, -1); for (int32_t i = 0; i < static_cast(sorted_decoded_vocab_.size()); ++i) { token_id_to_sorted_vocab_index_[sorted_decoded_vocab_[i].first] = i; } // The value means: the subtree is [i, trie_subtree_nodes_range[i]). trie_subtree_nodes_range_.resize(sorted_decoded_vocab_.size(), 0); std::stack> prefix_stack; for (size_t i = 0; i < sorted_decoded_vocab_.size(); ++i) { const auto& token = sorted_decoded_vocab_[i].second; while ((!prefix_stack.empty()) && (token.find(prefix_stack.top().first) == std::string::npos)) { const auto& top_pair = prefix_stack.top(); trie_subtree_nodes_range_[top_pair.second] = i; prefix_stack.pop(); } prefix_stack.push({token, i}); } while (!prefix_stack.empty()) { const auto& top_pair = prefix_stack.top(); trie_subtree_nodes_range_[top_pair.second] = sorted_decoded_vocab_.size(); prefix_stack.pop(); } } std::string TokenizerInfo::Impl::DumpMetadata() const { return DumpMetadataValue().serialize(false); } picojson::value TokenizerInfo::Impl::DumpMetadataValue() const { picojson::object obj; obj["vocab_type"] = picojson::value(static_cast(vocab_type_)); obj["vocab_size"] = picojson::value(static_cast(vocab_size_)); obj["add_prefix_space"] = picojson::value(add_prefix_space_); picojson::array stop_token_ids_array; for (auto id : stop_token_ids_) { stop_token_ids_array.push_back(picojson::value(static_cast(id))); } obj["stop_token_ids"] = picojson::value(std::move(stop_token_ids_array)); return picojson::value(std::move(obj)); } std::optional TokenizerInfo::Impl::CheckMetadataMatch( const picojson::value& metadata ) const { if (!metadata.is()) { return std::runtime_error("Expect an object"); } const auto& object = metadata.get(); if (object.find("vocab_type") == object.end()) { return std::runtime_error("Missing 'vocab_type' in metadata"); } auto vocab_type = object.at("vocab_type").get(); if (vocab_type != static_cast(vocab_type_)) { return std::runtime_error( "Vocab type mismatch: " + std::to_string(vocab_type) + " != " + std::to_string(static_cast(vocab_type_)) ); } if (object.find("vocab_size") == object.end()) { return std::runtime_error("Missing 'vocab_size' in metadata"); } auto vocab_size = object.at("vocab_size").get(); if (vocab_size != vocab_size_) { return std::runtime_error( "Vocab size mismatch: " + std::to_string(vocab_size) + " != " + std::to_string(vocab_size_) ); } if (object.find("add_prefix_space") == object.end()) { return std::runtime_error("Missing 'add_prefix_space' in metadata"); } auto add_prefix_space = object.at("add_prefix_space").get(); if (add_prefix_space != add_prefix_space_) { return std::runtime_error( "Add prefix space mismatch: " + std::to_string(add_prefix_space) + " != " + std::to_string(add_prefix_space_) ); } if (object.find("stop_token_ids") == object.end()) { return std::runtime_error("Missing 'stop_token_ids' in metadata"); } auto stop_token_ids = object.at("stop_token_ids").get(); std::vector stop_token_ids_vec; stop_token_ids_vec.reserve(stop_token_ids.size()); for (const auto& id : stop_token_ids) { if (!id.is()) { return std::runtime_error("Stop token id is not an integer"); } stop_token_ids_vec.push_back(static_cast(id.get())); } if (stop_token_ids_vec != stop_token_ids_) { return std::runtime_error("Stop token ids mismatch"); } return std::nullopt; } std::shared_ptr TokenizerInfo::Impl::FromVocabAndMetadata( const std::vector& encoded_vocab, const std::string& metadata ) { picojson::value v; std::string err = picojson::parse(v, metadata); XGRAMMAR_CHECK(err.empty()) << "Failed to parse metadata: " << err; const picojson::object& obj = v.get(); XGRAMMAR_CHECK(obj.count("vocab_type") && obj["vocab_type"].is()) << "Missing or invalid 'vocab_type' in metadata"; int vocab_type_int = static_cast(obj["vocab_type"].get()); XGRAMMAR_CHECK(vocab_type_int == 0 || vocab_type_int == 1 || vocab_type_int == 2) << "Invalid vocab_type in metadata: " << vocab_type_int; VocabType vocab_type = static_cast(vocab_type_int); XGRAMMAR_CHECK(obj.count("vocab_size") && obj["vocab_size"].is()) << "Missing or invalid 'vocab_size' in metadata"; int vocab_size = static_cast(obj["vocab_size"].get()); XGRAMMAR_CHECK(obj.count("add_prefix_space") && obj["add_prefix_space"].is()) << "Missing or invalid 'add_prefix_space' in metadata"; bool add_prefix_space = obj["add_prefix_space"].get(); std::vector stop_token_ids; XGRAMMAR_CHECK(obj.count("stop_token_ids") && obj["stop_token_ids"].is()) << "Missing or invalid 'stop_token_ids' in metadata"; for (const auto& id : obj["stop_token_ids"].get()) { XGRAMMAR_CHECK(id.is()) << "Stop token id is not an integer"; stop_token_ids.push_back(static_cast(id.get())); } return std::make_shared( encoded_vocab, vocab_type, vocab_size, stop_token_ids, add_prefix_space ); } std::string TokenizerInfo::Impl::DetectMetadataFromHF(const std::string& backend_str) { picojson::value v; std::string err = picojson::parse(v, backend_str); XGRAMMAR_CHECK(err.empty() && v.is()) << "Failed to parse JSON object: " << err; const picojson::object& obj = v.get(); VocabType vocab_type = HFTokenizerAnalyzer::DetectVocabType(obj); bool add_prefix_space = HFTokenizerAnalyzer::DetectAddPrefixSpace(obj); // Serialize the metadata picojson::object metadata_obj; metadata_obj["vocab_type"] = picojson::value(static_cast(vocab_type)); metadata_obj["add_prefix_space"] = picojson::value(add_prefix_space); return picojson::value(metadata_obj).serialize(false); } /************* TokenizerInfo *************/ TokenizerInfo::TokenizerInfo( const std::vector& encoded_vocab, VocabType vocab_type, std::optional vocab_size, std::optional> stop_token_ids, bool add_prefix_space ) : pimpl_(std::make_shared( encoded_vocab, vocab_type, vocab_size, stop_token_ids, add_prefix_space )) {} int TokenizerInfo::GetVocabSize() const { return pimpl_->GetVocabSize(); } VocabType TokenizerInfo::GetVocabType() const { return pimpl_->GetVocabType(); } bool TokenizerInfo::GetAddPrefixSpace() const { return pimpl_->GetAddPrefixSpace(); } const std::vector& TokenizerInfo::GetDecodedVocab() const { return pimpl_->GetDecodedVocab(); } const std::vector& TokenizerInfo::GetStopTokenIds() const { return pimpl_->GetStopTokenIds(); } const std::vector& TokenizerInfo::GetSpecialTokenIds() const { return pimpl_->GetSpecialTokenIds(); } const std::vector>& TokenizerInfo::GetSortedDecodedVocab() const { return pimpl_->GetSortedDecodedVocab(); } const std::vector& TokenizerInfo::GetTrieSubtreeNodesRange() const { return pimpl_->GetTrieSubtreeNodesRange(); } std::string TokenizerInfo::DumpMetadata() const { return pimpl_->DumpMetadata(); } TokenizerInfo TokenizerInfo::FromVocabAndMetadata( const std::vector& encoded_vocab, const std::string& metadata ) { return TokenizerInfo(Impl::FromVocabAndMetadata(encoded_vocab, metadata)); } std::string TokenizerInfo::DetectMetadataFromHF(const std::string& backend_str) { return Impl::DetectMetadataFromHF(backend_str); } std::string TokenizerInfo::SerializeJSON() const { return AutoSerializeJSON(*this, true); } std::variant TokenizerInfo::DeserializeJSON( const std::string& json_string ) { TokenizerInfo tokenizer_info{NullObj()}; if (auto err = AutoDeserializeJSON(&tokenizer_info, json_string, true, "TokenizerInfo")) { return err.value(); } return tokenizer_info; } } // namespace xgrammar xgrammar-0.2.3/cpp/tokenizer_info_impl.h000066400000000000000000000102751521764210300203410ustar00rootroot00000000000000#ifndef XGRAMMAR_TOKENIZER_INFO_IMPL_H_ #define XGRAMMAR_TOKENIZER_INFO_IMPL_H_ #include #include #include #include #include #include #include "support/reflection.h" #include "xgrammar/tokenizer_info.h" namespace xgrammar { class TokenizerInfo::Impl { public: explicit Impl() = default; Impl( const std::vector& encoded_vocab, VocabType vocab_type, std::optional vocab_size, std::optional> stop_token_ids, bool add_prefix_space ); VocabType GetVocabType() const { return vocab_type_; } bool GetAddPrefixSpace() const { return add_prefix_space_; } int GetVocabSize() const { return vocab_size_; } const std::vector& GetDecodedVocab() { return decoded_vocab_; } const std::vector& GetStopTokenIds() const { return stop_token_ids_; } const std::vector& GetSpecialTokenIds() const { return special_token_ids_; } const std::vector>& GetSortedDecodedVocab() const { return sorted_decoded_vocab_; } const std::vector& GetTrieSubtreeNodesRange() const { return trie_subtree_nodes_range_; } const std::vector& GetTokenIdToSortedVocabIndex() const { return token_id_to_sorted_vocab_index_; } std::string DumpMetadata() const; picojson::value DumpMetadataValue() const; static std::shared_ptr FromVocabAndMetadata( const std::vector& encoded_vocab, const std::string& metadata ); std::optional CheckMetadataMatch(const picojson::value& metadata) const; static std::string DetectMetadataFromHF(const std::string& backend_str); bool operator==(const Impl& other) const; private: static bool IsSpecialToken(const std::string& decoded_token); /*! \brief The vocabulary type. */ VocabType vocab_type_; /*! \brief The size of the vocabulary. */ int vocab_size_; /*! \brief Whether to add prefix space. */ bool add_prefix_space_; /*! \brief The vocabulary. Special tokens are included. */ std::vector decoded_vocab_; /*! \brief All (id, token) pairs sorted in lexicographic order. This sorting is done to * maximize prefix reuse during matching. Special tokens and stop tokens are not included. */ std::vector> sorted_decoded_vocab_; /*! \brief A pesudo-trie. trie_subtree_nodes_range[i] stores how many nodes there are in the * subtree. */ std::vector trie_subtree_nodes_range_; /*! \brief The stop tokens. When the GrammarMatcher can reach the end of the grammar, * stop tokens can be accepted. */ std::vector stop_token_ids_; /*! \brief The special tokens. These tokens are ignored (masked out) during the grammar-guided * generation. */ std::vector special_token_ids_; /*! \brief Reverse mapping: token_id -> index in sorted_decoded_vocab_. -1 if not present. */ std::vector token_id_to_sorted_vocab_index_; /*! * \brief The tokens used to detect stop tokens from the vocabulary. * * LLaMA2: * LLaMA3: <|end_of_text|>, <|eot_id|> * Phi-2: <|endoftext|> * Gemma: , * DeepSeek-V2: <｜end▁of▁sentence｜> */ inline static const std::unordered_set DETECTION_STOP_TOKENS = { "", "<|end_of_text|>", "<|eot_id|>", "<|endoftext|>", "", "<|eos|>", "", "<｜end▁of▁sentence｜>" }; friend struct member_trait; }; XGRAMMAR_MEMBER_TABLE( TokenizerInfo::Impl, "vocab_type", &TokenizerInfo::Impl::vocab_type_, "vocab_size", &TokenizerInfo::Impl::vocab_size_, "add_prefix_space", &TokenizerInfo::Impl::add_prefix_space_, "stop_token_ids", &TokenizerInfo::Impl::stop_token_ids_, "special_token_ids", &TokenizerInfo::Impl::special_token_ids_, "decoded_vocab", &TokenizerInfo::Impl::decoded_vocab_, "sorted_decoded_vocab", &TokenizerInfo::Impl::sorted_decoded_vocab_, "trie_subtree_nodes_range", &TokenizerInfo::Impl::trie_subtree_nodes_range_ ); } // namespace xgrammar #endif // XGRAMMAR_TOKENIZER_INFO_IMPL_H_ xgrammar-0.2.3/cpp/tvm_ffi/000077500000000000000000000000001521764210300155475ustar00rootroot00000000000000xgrammar-0.2.3/cpp/tvm_ffi/CMakeLists.txt000066400000000000000000000041371521764210300203140ustar00rootroot00000000000000find_package( Python COMPONENTS Interpreter Development.Module REQUIRED ) # Discover tvm_ffi from Python environment (used by scikit-build-core and standalone) if(NOT DEFINED tvm_ffi_ROOT) execute_process( COMMAND "${Python_EXECUTABLE}" -c "import tvm_ffi, os; print(os.path.join(os.path.dirname(tvm_ffi.__file__), 'share', 'cmake', 'tvm_ffi'))" OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE tvm_ffi_ROOT ) endif() find_package(tvm_ffi CONFIG REQUIRED) # Core logic used by bindings (no tvm_ffi dependency) add_library(python_methods STATIC python_methods.cc) target_include_directories(python_methods PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(python_methods PUBLIC xgrammar) # TVM-FFI shared library: C++ bindings entry point add_library(xgrammar_bindings SHARED tvm_ffi.cc) tvm_ffi_configure_target(xgrammar_bindings STUB_DIR "${PROJECT_SOURCE_DIR}/python" STUB_INIT ON) install(TARGETS xgrammar_bindings DESTINATION .) tvm_ffi_install(xgrammar_bindings DESTINATION .) target_include_directories(xgrammar_bindings PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) # Use tvm_ffi's include (and its DLPack) before project includes to satisfy tvm_ffi headers get_target_property(TVM_FFI_INCLUDES tvm_ffi::header INTERFACE_INCLUDE_DIRECTORIES) if(TVM_FFI_INCLUDES) target_include_directories(xgrammar_bindings BEFORE PRIVATE ${TVM_FFI_INCLUDES}) endif() target_link_libraries(xgrammar_bindings PRIVATE python_methods) target_link_libraries(xgrammar_bindings PRIVATE tvm_ffi::header tvm_ffi::shared) if(DEFINED SKBUILD_PROJECT_NAME) set(LIB_OUTPUT_DIRECTORY xgrammar) else() set(LIB_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/python/xgrammar) endif() set_target_properties(xgrammar_bindings PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${LIB_OUTPUT_DIRECTORY}) set_target_properties( xgrammar_bindings PROPERTIES LIBRARY_OUTPUT_DIRECTORY_DEBUG ${LIB_OUTPUT_DIRECTORY} ) set_target_properties( xgrammar_bindings PROPERTIES LIBRARY_OUTPUT_DIRECTORY_RELEASE ${LIB_OUTPUT_DIRECTORY} ) set_target_properties( xgrammar_bindings PROPERTIES LIBRARY_OUTPUT_DIRECTORY_REL_WITH_DEB_INFO ${LIB_OUTPUT_DIRECTORY} ) xgrammar-0.2.3/cpp/tvm_ffi/python_methods.cc000066400000000000000000000123311521764210300211220ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/tvm_ffi/python_methods.cc */ #include "python_methods.h" #include #include #include #include #include #include #include #include "../grammar_impl.h" #include "../support/logging.h" #include "../support/utils.h" #include "xgrammar/exception.h" namespace xgrammar { TokenizerInfo TokenizerInfo_Init( const std::vector& encoded_vocab, int vocab_type, std::optional vocab_size, std::optional> stop_token_ids, bool add_prefix_space ) { XGRAMMAR_CHECK(vocab_type == 0 || vocab_type == 1 || vocab_type == 2) << "Invalid vocab type: " << vocab_type; return TokenizerInfo( encoded_vocab, static_cast(vocab_type), vocab_size, stop_token_ids, add_prefix_space ); } int TokenizerInfo_GetVocabType(const TokenizerInfo& tokenizer) { return static_cast(tokenizer.GetVocabType()); } std::vector Testing_DebugGetMaskedTokensFromBitmask( intptr_t token_bitmask_ptr, std::vector shape, int32_t vocab_size, int32_t index ) { XGRAMMAR_CHECK(shape.size() == 1 || shape.size() == 2) << "token_bitmask tensor must be 1D or 2D"; DLTensor bitmask_dltensor{ reinterpret_cast(token_bitmask_ptr), DLDevice{kDLCPU, 0}, static_cast(shape.size()), GetBitmaskDLType(), shape.data(), nullptr, 0 }; std::vector result; _DebugGetMaskedTokensFromBitmask(&result, bitmask_dltensor, vocab_size, index); return result; } std::pair Testing_IsSingleTokenBitmask( intptr_t token_bitmask_ptr, std::vector shape, int32_t vocab_size, int32_t index ) { XGRAMMAR_CHECK(shape.size() == 1 || shape.size() == 2) << "token_bitmask tensor must be 1D or 2D"; DLTensor bitmask_dltensor{ reinterpret_cast(token_bitmask_ptr), DLDevice{kDLCPU, 0}, static_cast(shape.size()), GetBitmaskDLType(), shape.data(), nullptr, 0 }; return _IsSingleTokenBitmask(bitmask_dltensor, vocab_size, index); } void Kernels_ApplyTokenBitmaskInplaceCPU( intptr_t logits_ptr, std::pair logits_shape, std::pair logits_strides, intptr_t bitmask_ptr, std::pair bitmask_shape, std::pair bitmask_strides, int vocab_size, std::optional> indices, std::string logit_type ) { std::array logits_shape_arr = {logits_shape.first, logits_shape.second}; std::array logits_strides_arr = {logits_strides.first, logits_strides.second}; std::array bitmask_shape_arr = {bitmask_shape.first, bitmask_shape.second}; std::array bitmask_strides_arr = {bitmask_strides.first, bitmask_strides.second}; DLDataType logit_dtype; if (logit_type == "float32") { logit_dtype = DLDataType{kDLFloat, 32, 1}; } else if (logit_type == "float16") { logit_dtype = DLDataType{kDLFloat, 16, 1}; } else if (logit_type == "bfloat16") { logit_dtype = DLDataType{kDLBfloat, 16, 1}; } else { XGRAMMAR_LOG(FATAL) << "Unsupported logit type: " << logit_type; } DLTensor logits_dltensor{ reinterpret_cast(logits_ptr), DLDevice{kDLCPU, 0}, 2, logit_dtype, logits_shape_arr.data(), logits_strides_arr.data(), 0 }; DLTensor bitmask_dltensor{ reinterpret_cast(bitmask_ptr), DLDevice{kDLCPU, 0}, 2, GetBitmaskDLType(), bitmask_shape_arr.data(), bitmask_strides_arr.data(), 0 }; ApplyTokenBitmaskInplaceCPU(&logits_dltensor, bitmask_dltensor, vocab_size, indices); } std::vector GetAllowEmptyRuleIds(const CompiledGrammar& compiled_grammar) { return compiled_grammar.GetGrammar()->allow_empty_rule_ids; } Grammar Grammar_FromStructuralTag( const std::string& structural_tag_json, const std::optional& tokenizer_info ) { auto result = Grammar::FromStructuralTag(structural_tag_json, tokenizer_info); if (std::holds_alternative(result)) { ThrowVariantError(std::get(result)); } return std::get(result); } Grammar Grammar_DeserializeJSON(const std::string& json_string) { auto result = Grammar::DeserializeJSON(json_string); if (std::holds_alternative(result)) { ThrowVariantError(std::get(result)); } return std::get(result); } TokenizerInfo TokenizerInfo_DeserializeJSON(const std::string& json_string) { auto result = TokenizerInfo::DeserializeJSON(json_string); if (std::holds_alternative(result)) { ThrowVariantError(std::get(result)); } return std::get(result); } CompiledGrammar CompiledGrammar_DeserializeJSON( const std::string& json_string, const TokenizerInfo& tokenizer ) { auto result = CompiledGrammar::DeserializeJSON(json_string, tokenizer); if (std::holds_alternative(result)) { ThrowVariantError(std::get(result)); } return std::get(result); } } // namespace xgrammar xgrammar-0.2.3/cpp/tvm_ffi/python_methods.h000066400000000000000000000036531521764210300207730ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/tvm_ffi/python_methods.h * \brief The header for the support of grammar-guided generation. */ #ifndef XGRAMMAR_NANOBIND_PYTHON_METHODS_H_ #define XGRAMMAR_NANOBIND_PYTHON_METHODS_H_ #include #include #include #include #include #include "xgrammar/tokenizer_info.h" namespace xgrammar { TokenizerInfo TokenizerInfo_Init( const std::vector& encoded_vocab, int vocab_type, std::optional vocab_size, std::optional> stop_token_ids, bool add_prefix_space ); int TokenizerInfo_GetVocabType(const TokenizerInfo& tokenizer); std::vector Testing_DebugGetMaskedTokensFromBitmask( intptr_t token_bitmask_ptr, std::vector shape, int32_t vocab_size, int32_t index ); std::pair Testing_IsSingleTokenBitmask( intptr_t token_bitmask_ptr, std::vector shape, int32_t vocab_size, int32_t index ); void Kernels_ApplyTokenBitmaskInplaceCPU( intptr_t logits_ptr, std::pair logits_shape, std::pair logits_strides, intptr_t bitmask_ptr, std::pair bitmask_shape, std::pair bitmask_strides, int vocab_size, std::optional> indices, std::string logit_type ); std::vector GetAllowEmptyRuleIds(const CompiledGrammar& compiled_grammar); Grammar Grammar_FromStructuralTag( const std::string& structural_tag_json, const std::optional& tokenizer_info = std::nullopt ); Grammar Grammar_DeserializeJSON(const std::string& json_string); TokenizerInfo TokenizerInfo_DeserializeJSON(const std::string& json_string); CompiledGrammar CompiledGrammar_DeserializeJSON( const std::string& json_string, const TokenizerInfo& tokenizer ); } // namespace xgrammar #endif // XGRAMMAR_NANOBIND_PYTHON_METHODS_H_ xgrammar-0.2.3/cpp/tvm_ffi/tvm_ffi.cc000066400000000000000000001107721521764210300175200ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/tvm_ffi/tvm_ffi.cc * \brief TVM-FFI bindings for xgrammar. */ #include #include #include #include #include #include #include #include #include #include #include #include #include "../grammar_functor.h" #include "../json_schema_converter.h" #include "../regex_converter.h" #include "../support/utils.h" #include "../testing.h" #include "python_methods.h" #include "xgrammar/exception.h" #include "xgrammar/matcher.h" namespace ffi = tvm::ffi; namespace refl = tvm::ffi::reflection; namespace xgrammar { // ----- Error handling ----- // ----- Helpers: convert FFI types to/from xgrammar types ----- static std::string BytesToString(const tvm::ffi::Bytes& bytes) { std::string result(bytes.data(), bytes.size()); return result; } // Convert ffi::Array to vector; each element can be str or bytes (like // accept_string). static std::vector ArrayAnyToVectorString(ffi::Array array) { std::vector result; result.reserve(static_cast(array.size())); for (int64_t i = 0; i < static_cast(array.size()); ++i) { ffi::AnyView view = array[i]; if (view.as()) { result.push_back(BytesToString(view.cast())); } else if (view.as()) { result.push_back(view.cast()); } else { TVM_FFI_THROW(RuntimeError) << "Unsupported type in encoded_vocab: expected str or bytes"; XGRAMMAR_UNREACHABLE(); } } return result; } static ffi::Array VectorStringToBytesArray(const std::vector& string_vector ) { ffi::Array bytes_array; for (const auto& value : string_vector) { bytes_array.push_back(ffi::Bytes(value)); } return bytes_array; } static std::optional OptionalIntFromView(ffi::AnyView v) { if (v == nullptr) return std::nullopt; return v.cast(); } static std::optional OptionalBoolFromView(ffi::AnyView v) { if (v == nullptr) return std::nullopt; return static_cast(v.cast()); } static std::optional> OptionalInt32VectorFromView(ffi::AnyView v) { if (v == nullptr) return std::nullopt; ffi::Array array = v.cast>(); std::vector result; result.reserve(static_cast(array.size())); for (int64_t i = 0; i < static_cast(array.size()); ++i) result.push_back(static_cast(array[i])); return result; } static std::optional> OptionalIntVectorFromView(ffi::AnyView v) { if (v == nullptr) return std::nullopt; ffi::Array array = v.cast>(); std::vector result; result.reserve(static_cast(array.size())); for (int64_t i = 0; i < static_cast(array.size()); ++i) result.push_back(static_cast(array[i])); return result; } static std::optional> OptionalSeparatorsFromView(ffi::AnyView v ) { if (v == nullptr) return std::nullopt; ffi::Array separators_array = v.cast>(); if (separators_array.size() < 2) return std::nullopt; return std::make_pair(separators_array[0], separators_array[1]); } static std::variant ParseMaxThreads(ffi::AnyView max_threads_view) { if (max_threads_view == nullptr) return "auto"; auto int_value = max_threads_view.as(); if (int_value.has_value()) { return static_cast(int_value.value()); } auto string_value = max_threads_view.as(); if (string_value.has_value()) { return string_value.value(); } TVM_FFI_THROW(RuntimeError) << "Invalid max_threads value"; XGRAMMAR_UNREACHABLE(); } // Wrap std::exception into TVM-FFI error #define XGRAMMAR_FFI_TRY_BEGIN() try { #define XGRAMMAR_FFI_TRY_END() \ } \ catch (const XGrammarError& e) { \ throw ffi::Error(e.GetType(), e.what(), ""); \ XGRAMMAR_UNREACHABLE(); \ } // ----- Object wrappers (hold xgrammar types, inherit ffi::Object) ----- class TokenizerInfoObj : public ffi::Object { public: TokenizerInfo value; explicit TokenizerInfoObj(TokenizerInfo v) : value(std::move(v)) {} TokenizerInfoObj( ffi::Array encoded_vocab, int64_t vocab_type, ffi::AnyView vocab_size_opt, ffi::AnyView stop_token_ids_opt, bool add_prefix_space ) : value(NullObj{}) { XGRAMMAR_FFI_TRY_BEGIN(); value = TokenizerInfo_Init( ArrayAnyToVectorString(encoded_vocab), static_cast(vocab_type), OptionalIntFromView(vocab_size_opt), OptionalInt32VectorFromView(stop_token_ids_opt), add_prefix_space ); XGRAMMAR_FFI_TRY_END(); } static constexpr bool _type_mutable = false; TVM_FFI_DECLARE_OBJECT_INFO_FINAL( "xgrammar.tvm_ffi_binding.TokenizerInfo", TokenizerInfoObj, ffi::Object ); }; class GrammarObj : public ffi::Object { public: Grammar value; explicit GrammarObj(Grammar v) : value(std::move(v)) {} static constexpr bool _type_mutable = false; TVM_FFI_DECLARE_OBJECT_INFO_FINAL("xgrammar.tvm_ffi_binding.Grammar", GrammarObj, ffi::Object); }; class CompiledGrammarObj : public ffi::Object { public: CompiledGrammar value; explicit CompiledGrammarObj(CompiledGrammar v) : value(std::move(v)) {} static constexpr bool _type_mutable = false; TVM_FFI_DECLARE_OBJECT_INFO_FINAL( "xgrammar.tvm_ffi_binding.CompiledGrammar", CompiledGrammarObj, ffi::Object ); }; class GrammarCompilerObj : public ffi::Object { public: GrammarCompiler value; explicit GrammarCompilerObj(GrammarCompiler v) : value(std::move(v)) {} GrammarCompilerObj( ffi::ObjectRef tokenizer_ref, int64_t max_threads, bool cache_enabled, int64_t max_memory_bytes ) : value( tokenizer_ref.as()->value, static_cast(max_threads), cache_enabled, max_memory_bytes ) {} static constexpr bool _type_mutable = true; TVM_FFI_DECLARE_OBJECT_INFO_FINAL( "xgrammar.tvm_ffi_binding.GrammarCompiler", GrammarCompilerObj, ffi::Object ); }; class GrammarMatcherObj : public ffi::Object { public: GrammarMatcher value; explicit GrammarMatcherObj(GrammarMatcher v) : value(std::move(v)) {} GrammarMatcherObj( ffi::ObjectRef compiled_grammar_ref, ffi::AnyView override_stop_tokens_opt, bool terminate_without_stop_token, int64_t max_rollback_tokens ) : value( compiled_grammar_ref.as()->value, OptionalIntVectorFromView(override_stop_tokens_opt), terminate_without_stop_token, static_cast(max_rollback_tokens) ) {} static constexpr bool _type_mutable = true; TVM_FFI_DECLARE_OBJECT_INFO_FINAL( "xgrammar.tvm_ffi_binding.GrammarMatcher", GrammarMatcherObj, ffi::Object ); }; class BatchGrammarMatcherObj : public ffi::Object { public: BatchGrammarMatcher value; BatchGrammarMatcherObj() = default; explicit BatchGrammarMatcherObj(BatchGrammarMatcher v) : value(std::move(v)) {} explicit BatchGrammarMatcherObj(ffi::AnyView max_threads_view) : value(ParseMaxThreads(max_threads_view)) {} static constexpr bool _type_mutable = true; TVM_FFI_DECLARE_OBJECT_INFO_FINAL( "xgrammar.tvm_ffi_binding.BatchGrammarMatcher", BatchGrammarMatcherObj, ffi::Object ); }; // ----- Registration: ObjectDef ----- // Custom constructors are handled via lambda wrappers below; TVM-FFI uses refl::init() // and we need to bridge FFI types to xgrammar types in those lambdas. TVM_FFI_STATIC_INIT_BLOCK() { using O = ffi::ObjectRef; // TokenizerInfo: init(encoded_vocab, vocab_type, vocab_size_opt, stop_token_ids_opt, // add_prefix_space) refl::ObjectDef() .def(refl::init, int64_t, ffi::AnyView, ffi::AnyView, bool>()) .def( "vocab_type", [](const TokenizerInfoObj* o) { return static_cast(TokenizerInfo_GetVocabType(o->value)); } ) .def( "vocab_size", [](const TokenizerInfoObj* o) { return static_cast(o->value.GetVocabSize()); } ) .def( "add_prefix_space", [](const TokenizerInfoObj* o) { return o->value.GetAddPrefixSpace(); } ) .def( "decoded_vocab", [](const TokenizerInfoObj* o) { const auto& decoded_vocab = o->value.GetDecodedVocab(); return VectorStringToBytesArray(decoded_vocab); } ) .def( "stop_token_ids", [](const TokenizerInfoObj* o) { const auto& stop_token_ids = o->value.GetStopTokenIds(); ffi::Array stop_token_ids_array; for (int32_t token_id : stop_token_ids) stop_token_ids_array.push_back(static_cast(token_id)); return stop_token_ids_array; } ) .def( "special_token_ids", [](const TokenizerInfoObj* o) { const auto& special_token_ids = o->value.GetSpecialTokenIds(); ffi::Array special_token_ids_array; for (int32_t token_id : special_token_ids) special_token_ids_array.push_back(static_cast(token_id)); return special_token_ids_array; } ) .def( "dump_metadata", [](const TokenizerInfoObj* o) { return ffi::String(o->value.DumpMetadata()); } ) .def_static( "from_vocab_and_metadata", [](ffi::Array encoded_vocab, ffi::String metadata) { XGRAMMAR_FFI_TRY_BEGIN(); auto v = TokenizerInfo::FromVocabAndMetadata( ArrayAnyToVectorString(encoded_vocab), metadata ); return ffi::ObjectRef(ffi::make_object(std::move(v))); XGRAMMAR_FFI_TRY_END(); } ) .def_static( "_detect_metadata_from_hf", [](ffi::String backend_str) { return ffi::String(TokenizerInfo::DetectMetadataFromHF(backend_str)); } ) .def( "serialize_json", [](const TokenizerInfoObj* o) { return ffi::String(o->value.SerializeJSON()); } ) .def_static("deserialize_json", [](ffi::String json_string) { XGRAMMAR_FFI_TRY_BEGIN(); auto r = TokenizerInfo::DeserializeJSON(json_string); if (std::holds_alternative(r)) { const auto& err = std::get(r); throw ffi::Error(GetTypeFromVariantError(err), GetMessageFromVariantError(err), ""); } return ffi::ObjectRef(ffi::make_object(std::get(r))); XGRAMMAR_FFI_TRY_END(); }); // Grammar refl::ObjectDef() .def("to_string", [](const GrammarObj* o) { return ffi::String(o->value.ToString()); }) .def_static( "from_ebnf", [](ffi::String ebnf_str, ffi::String root_rule_name) { return ffi::ObjectRef( ffi::make_object(Grammar::FromEBNF(ebnf_str, root_rule_name)) ); } ) .def_static( "from_json_schema", [](ffi::String schema, bool any_whitespace, ffi::AnyView indent, ffi::AnyView separators, bool strict_mode, ffi::AnyView max_whitespace_cnt, bool print_converted_ebnf, bool any_order) { XGRAMMAR_FFI_TRY_BEGIN(); auto g = Grammar::FromJSONSchema( schema, any_whitespace, OptionalIntFromView(indent), OptionalSeparatorsFromView(separators), strict_mode, OptionalIntFromView(max_whitespace_cnt), print_converted_ebnf, any_order ); return ffi::ObjectRef(ffi::make_object(std::move(g))); XGRAMMAR_FFI_TRY_END(); } ) .def_static( "from_regex", [](ffi::String regex, bool print_converted_ebnf) { return ffi::ObjectRef( ffi::make_object(Grammar::FromRegex(regex, print_converted_ebnf)) ); } ) .def_static( "from_structural_tag", [](ffi::String structural_tag_json) { XGRAMMAR_FFI_TRY_BEGIN(); Grammar grammar = Grammar_FromStructuralTag(structural_tag_json); return ffi::ObjectRef(ffi::make_object(std::move(grammar))); XGRAMMAR_FFI_TRY_END(); } ) .def_static( "builtin_json_grammar", []() { return ffi::ObjectRef(ffi::make_object(Grammar::BuiltinJSONGrammar())); } ) .def_static( "union", [](ffi::Array grammars) { std::vector grammar_list; grammar_list.reserve(static_cast(grammars.size())); for (int64_t i = 0; i < static_cast(grammars.size()); ++i) { grammar_list.push_back(grammars[i].as()->value); } return ffi::ObjectRef(ffi::make_object(Grammar::Union(grammar_list))); } ) .def_static( "concat", [](ffi::Array grammars) { std::vector grammar_list; grammar_list.reserve(static_cast(grammars.size())); for (int64_t i = 0; i < static_cast(grammars.size()); ++i) { grammar_list.push_back(grammars[i].as()->value); } return ffi::ObjectRef(ffi::make_object(Grammar::Concat(grammar_list))); } ) .def( "serialize_json", [](const GrammarObj* o) { return ffi::String(o->value.SerializeJSON()); } ) .def_static("deserialize_json", [](ffi::String json_string) { XGRAMMAR_FFI_TRY_BEGIN(); Grammar grammar = Grammar_DeserializeJSON(json_string); return ffi::ObjectRef(ffi::make_object(std::move(grammar))); XGRAMMAR_FFI_TRY_END(); }); // CompiledGrammar refl::ObjectDef() .def( "grammar", [](const CompiledGrammarObj* o) { return ffi::ObjectRef(ffi::make_object(o->value.GetGrammar())); } ) .def( "tokenizer_info", [](const CompiledGrammarObj* o) { return ffi::ObjectRef(ffi::make_object(o->value.GetTokenizerInfo())); } ) .def( "memory_size_bytes", [](const CompiledGrammarObj* o) { return static_cast(o->value.MemorySizeBytes()); } ) .def( "serialize_json", [](const CompiledGrammarObj* o) { return ffi::String(o->value.SerializeJSON()); } ) .def_static("deserialize_json", [](ffi::String json_string, O tokenizer_ref) { XGRAMMAR_FFI_TRY_BEGIN(); const TokenizerInfo& tokenizer_info = tokenizer_ref.as()->value; CompiledGrammar compiled_grammar = CompiledGrammar_DeserializeJSON(json_string, tokenizer_info); return ffi::ObjectRef(ffi::make_object(std::move(compiled_grammar))); XGRAMMAR_FFI_TRY_END(); }); // GrammarCompiler: init(tokenizer_info, max_threads, cache_enabled, max_memory_bytes) refl::ObjectDef() .def(refl::init()) .def( "compile_json_schema", [](GrammarCompilerObj* o, ffi::String schema, bool any_whitespace, ffi::AnyView indent, ffi::AnyView separators, bool strict_mode, ffi::AnyView max_whitespace_cnt, bool any_order) { XGRAMMAR_FFI_TRY_BEGIN(); CompiledGrammar cg = o->value.CompileJSONSchema( schema, any_whitespace, OptionalIntFromView(indent), OptionalSeparatorsFromView(separators), strict_mode, OptionalIntFromView(max_whitespace_cnt), any_order ); return ffi::ObjectRef(ffi::make_object(std::move(cg))); XGRAMMAR_FFI_TRY_END(); } ) .def( "compile_builtin_json_grammar", [](GrammarCompilerObj* o) { return ffi::ObjectRef( ffi::make_object(o->value.CompileBuiltinJSONGrammar()) ); } ) .def( "compile_structural_tag", [](GrammarCompilerObj* o, ffi::String structural_tag_json) { return ffi::ObjectRef(ffi::make_object( o->value.CompileStructuralTag(structural_tag_json) )); } ) .def( "compile_regex", [](GrammarCompilerObj* o, ffi::String regex) { return ffi::ObjectRef(ffi::make_object(o->value.CompileRegex(regex)) ); } ) .def( "compile_grammar_ebnf", [](GrammarCompilerObj* o, O grammar_ref) { return ffi::ObjectRef(ffi::make_object( o->value.CompileGrammar(grammar_ref.as()->value) )); } ) .def( "compile_grammar_from_strings", [](GrammarCompilerObj* o, ffi::String ebnf_str, ffi::String root_rule_name) { return ffi::ObjectRef(ffi::make_object( o->value.CompileGrammar(ebnf_str, root_rule_name) )); } ) .def("clear_cache", [](GrammarCompilerObj* o) { o->value.ClearCache(); }) .def( "get_cache_size_bytes", [](const GrammarCompilerObj* o) { return o->value.GetCacheSizeBytes(); } ) .def("cache_limit_bytes", [](const GrammarCompilerObj* o) { return o->value.CacheLimitBytes(); }); // BatchGrammarMatcher: init(max_threads) refl::ObjectDef() .def(refl::init()) .def( "batch_fill_next_token_bitmask", [](BatchGrammarMatcherObj* o, ffi::Array matchers_ref, ffi::AnyView batch_token_bitmask, ffi::AnyView indices, bool debug_print) { std::vector matchers; matchers.reserve(matchers_ref.size()); for (int64_t i = 0; i < static_cast(matchers_ref.size()); ++i) { matchers.push_back(matchers_ref[i].as()->value); } DLTensor* bitmask = batch_token_bitmask.cast(); o->value.BatchFillNextTokenBitmask( &matchers, bitmask, OptionalInt32VectorFromView(indices), debug_print ); } ) .def_static( "batch_accept_string", [](ffi::Array matchers_ref, ffi::Array input_str_byte_union, bool debug_print ) { std::vector matchers; matchers.reserve(matchers_ref.size()); for (int64_t i = 0; i < static_cast(matchers_ref.size()); ++i) { matchers.push_back(matchers_ref[i].as()->value); } std::vector input_strings = ArrayAnyToVectorString(input_str_byte_union); std::vector acceptance_results = BatchGrammarMatcher::BatchAcceptString(&matchers, input_strings, debug_print); ffi::Array acceptance_results_array; for (uint8_t acceptance_flag : acceptance_results) acceptance_results_array.push_back(static_cast(acceptance_flag)); return acceptance_results_array; } ) .def_static( "batch_accept_token", [](ffi::Array matchers_ref, ffi::Array token_ids, bool debug_print) { std::vector matchers; matchers.reserve(matchers_ref.size()); for (int64_t i = 0; i < static_cast(matchers_ref.size()); ++i) { matchers.push_back(matchers_ref[i].as()->value); } std::vector token_id_vector; token_id_vector.reserve(token_ids.size()); for (int64_t i = 0; i < static_cast(token_ids.size()); ++i) { token_id_vector.push_back(static_cast(token_ids[i])); } std::vector acceptance_results = BatchGrammarMatcher::BatchAcceptToken(&matchers, token_id_vector, debug_print); ffi::Array acceptance_results_array; for (uint8_t acceptance_flag : acceptance_results) acceptance_results_array.push_back(static_cast(acceptance_flag)); return acceptance_results_array; } ) .def_static("batch_rollback", [](ffi::Array matchers_ref, ffi::Array num_tokens) { std::vector matchers; matchers.reserve(matchers_ref.size()); for (int64_t i = 0; i < static_cast(matchers_ref.size()); ++i) { matchers.push_back(matchers_ref[i].as()->value); } std::vector num_tokens_vector; num_tokens_vector.reserve(num_tokens.size()); for (int64_t i = 0; i < static_cast(num_tokens.size()); ++i) { num_tokens_vector.push_back(static_cast(num_tokens[i])); } BatchGrammarMatcher::BatchRollback(&matchers, num_tokens_vector); }); // GrammarMatcher: init(compiled_grammar, override_stop_tokens_opt, terminate_without_stop, // max_rollback_tokens) refl::ObjectDef() .def(refl::init()) .def( "accept_token", [](GrammarMatcherObj* o, int64_t token_id, bool debug_print) { return o->value.AcceptToken(static_cast(token_id), debug_print); } ) .def( "accept_string", [](GrammarMatcherObj* o, ffi::Any input_str_bytes_union, bool debug_print) { ffi::AnyView view = input_str_bytes_union; if (view.as()) { return o->value.AcceptString(BytesToString(view.cast()), debug_print); } else if (view.as()) { return o->value.AcceptString(view.cast(), debug_print); } else { TVM_FFI_THROW(RuntimeError) << "Unsupported type in accept_string"; XGRAMMAR_UNREACHABLE(); } } ) .def( "fill_next_token_bitmask", [](GrammarMatcherObj* o, ffi::AnyView token_bitmask, int64_t index, bool debug_print) { DLTensor* dlt = token_bitmask.cast(); return o->value.FillNextTokenBitmask(dlt, static_cast(index), debug_print); } ) .def( "traverse_draft_tree", [](GrammarMatcherObj* o, ffi::AnyView retrieve_next_token, ffi::AnyView retrieve_next_sibling, ffi::AnyView draft_tokens, ffi::AnyView token_bitmask, double time_threshold) { DLTensor* retrieve_next_token_ptr = retrieve_next_token.cast(); DLTensor* retrieve_next_sibling_ptr = retrieve_next_sibling.cast(); DLTensor* draft_tokens_ptr = draft_tokens.cast(); DLTensor* token_bitmask_ptr = token_bitmask.cast(); return o->value.TraverseDraftTree( retrieve_next_token_ptr, retrieve_next_sibling_ptr, draft_tokens_ptr, token_bitmask_ptr, time_threshold ); } ) .def( "find_jump_forward_string", [](GrammarMatcherObj* o) { return ffi::String(o->value.FindJumpForwardString()); } ) .def( "rollback", [](GrammarMatcherObj* o, int64_t num_tokens) { o->value.Rollback(static_cast(num_tokens)); } ) .def( "fork", [](GrammarMatcherObj* o) { return ffi::ObjectRef(ffi::make_object(o->value.Fork())); } ) .def("is_terminated", [](const GrammarMatcherObj* o) { return o->value.IsTerminated(); }) .def("is_completed", [](const GrammarMatcherObj* o) { return o->value.IsCompleted(); }) .def("reset", [](GrammarMatcherObj* o) { o->value.Reset(); }) .def( "max_rollback_tokens", [](const GrammarMatcherObj* o) { return static_cast(o->value.GetMaxRollbackTokens()); } ) .def( "stop_token_ids", [](const GrammarMatcherObj* o) { const auto& stop_token_ids = o->value.GetStopTokenIds(); ffi::Array stop_token_ids_array; for (int token_id : stop_token_ids) stop_token_ids_array.push_back(static_cast(token_id)); return stop_token_ids_array; } ) .def("_debug_print_internal_state", [](const GrammarMatcherObj* o) { return ffi::String(o->value._DebugPrintInternalState()); }); // ----- Global functions: testing, kernels, config, exceptions ----- refl::GlobalDef() .def( "xgrammar.tvm_ffi_binding.testing._json_schema_to_ebnf", [](ffi::String schema, bool any_whitespace, ffi::AnyView indent, ffi::AnyView separators, bool strict_mode, ffi::AnyView max_whitespace_cnt, bool any_order) { return ffi::String(JSONSchemaToEBNF( schema, any_whitespace, OptionalIntFromView(indent), OptionalSeparatorsFromView(separators), strict_mode, OptionalIntFromView(max_whitespace_cnt), JSONFormat::kJSON, any_order )); } ) .def( "xgrammar.tvm_ffi_binding.testing._regex_to_ebnf", [](ffi::String regex, ffi::AnyView with_rule_name_opt) { bool with_rule_name = OptionalBoolFromView(with_rule_name_opt).value_or(true); return ffi::String(RegexToEBNF(regex, with_rule_name)); } ) .def( "xgrammar.tvm_ffi_binding.testing._ebnf_to_grammar_no_normalization", [](ffi::String ebnf_str, ffi::String root_rule_name) { Grammar grammar = _EBNFToGrammarNoNormalization(ebnf_str, root_rule_name); return ffi::ObjectRef(ffi::make_object(std::move(grammar))); } ) .def( "xgrammar.tvm_ffi_binding.testing._get_masked_tokens_from_bitmask", [](int64_t token_bitmask_ptr, ffi::Array shape, int64_t vocab_size, int64_t index ) { std::vector shape_vector; for (int64_t i = 0; i < static_cast(shape.size()); ++i) shape_vector.push_back(shape[i]); std::vector masked_tokens = Testing_DebugGetMaskedTokensFromBitmask( static_cast(token_bitmask_ptr), shape_vector, static_cast(vocab_size), static_cast(index) ); ffi::Array masked_tokens_array; for (int token_id : masked_tokens) masked_tokens_array.push_back(static_cast(token_id)); return masked_tokens_array; } ) .def( "xgrammar.tvm_ffi_binding.testing._is_single_token_bitmask", [](int64_t token_bitmask_ptr, ffi::Array shape, int64_t vocab_size, int64_t index ) { std::vector shape_vector; for (int64_t i = 0; i < static_cast(shape.size()); ++i) shape_vector.push_back(shape[i]); auto single_token_result = Testing_IsSingleTokenBitmask( static_cast(token_bitmask_ptr), shape_vector, static_cast(vocab_size), static_cast(index) ); return ffi::Array{ static_cast(single_token_result.first), static_cast(single_token_result.second) }; } ) .def( "xgrammar.tvm_ffi_binding.testing._get_allow_empty_rule_ids", [](O compiled_grammar_ref) { const auto& compiled_grammar = compiled_grammar_ref.as()->value; std::vector allow_empty_rule_ids = GetAllowEmptyRuleIds(compiled_grammar); ffi::Array allow_empty_rule_ids_array; for (int32_t rule_id : allow_empty_rule_ids) allow_empty_rule_ids_array.push_back(static_cast(rule_id)); return allow_empty_rule_ids_array; } ) .def( "xgrammar.tvm_ffi_binding.testing._generate_range_regex", [](ffi::AnyView start, ffi::AnyView end) { std::optional start_value = OptionalIntFromView(start); std::optional end_value = OptionalIntFromView(end); std::string regex_string = GenerateRangeRegex(start_value, end_value); regex_string.erase( std::remove(regex_string.begin(), regex_string.end(), '\0'), regex_string.end() ); return ffi::String(regex_string); } ) .def( "xgrammar.tvm_ffi_binding.testing._generate_float_regex", [](ffi::AnyView start, ffi::AnyView end, bool exclusive_start, bool exclusive_end) { std::optional start_value = start == nullptr ? std::nullopt : std::make_optional(start.cast()); std::optional end_value = end == nullptr ? std::nullopt : std::make_optional(end.cast()); std::string regex_string = GenerateFloatRangeRegex(start_value, end_value, exclusive_start, exclusive_end); regex_string.erase( std::remove(regex_string.begin(), regex_string.end(), '\0'), regex_string.end() ); return ffi::String(regex_string); } ) .def( "xgrammar.tvm_ffi_binding.testing._qwen_xml_tool_calling_to_ebnf", [](ffi::String schema, bool any_order) { return ffi::String(QwenXMLToolCallingToEBNF(schema, any_order)); } ) .def( "xgrammar.tvm_ffi_binding.testing._minimax_xml_tool_calling_to_ebnf", [](ffi::String schema, bool any_order) { return ffi::String(MiniMaxXMLToolCallingToEBNF(schema, any_order)); } ) .def( "xgrammar.tvm_ffi_binding.testing._deepseek_xml_tool_calling_to_ebnf", [](ffi::String schema, bool any_order) { return ffi::String(DeepSeekXMLToolCallingToEBNF(schema, any_order)); } ) .def( "xgrammar.tvm_ffi_binding.testing._glm_xml_tool_calling_to_ebnf", [](ffi::String schema, bool any_order) { return ffi::String(GlmXMLToolCallingToEBNF(schema, any_order)); } ) .def( "xgrammar.tvm_ffi_binding.testing._print_grammar_fsms", [](O grammar_ref) { return ffi::String(_PrintGrammarFSMs(grammar_ref.as()->value)); } ) .def( "xgrammar.tvm_ffi_binding.testing.grammar_functor.structure_normalizer", [](O grammar_ref) { return ffi::ObjectRef(ffi::make_object( StructureNormalizer::Apply(grammar_ref.as()->value) )); } ) .def( "xgrammar.tvm_ffi_binding.testing.grammar_functor.byte_string_fuser", [](O grammar_ref) { return ffi::ObjectRef(ffi::make_object( ByteStringFuser::Apply(grammar_ref.as()->value) )); } ) .def( "xgrammar.tvm_ffi_binding.testing.grammar_functor.rule_inliner", [](O grammar_ref) { return ffi::ObjectRef(ffi::make_object( RuleInliner::Apply(grammar_ref.as()->value) )); } ) .def( "xgrammar.tvm_ffi_binding.testing.grammar_functor.dead_code_eliminator", [](O grammar_ref) { return ffi::ObjectRef(ffi::make_object( DeadCodeEliminator::Apply(grammar_ref.as()->value) )); } ) .def( "xgrammar.tvm_ffi_binding.testing.grammar_functor.lookahead_assertion_analyzer", [](O grammar_ref) { return ffi::ObjectRef(ffi::make_object( LookaheadAssertionAnalyzer::Apply(grammar_ref.as()->value) )); } ) .def( "xgrammar.tvm_ffi_binding.testing.grammar_functor.grammar_optimizer", [](O grammar_ref) { return ffi::ObjectRef(ffi::make_object( GrammarOptimizer::Apply(grammar_ref.as()->value) )); } ) .def( "xgrammar.tvm_ffi_binding.testing.grammar_functor.repetition_normalizer", [](O grammar_ref) { Grammar grammar = grammar_ref.as()->value; RepetitionNormalizer::Apply(&grammar); return ffi::ObjectRef(ffi::make_object(std::move(grammar))); } ) .def( "xgrammar.tvm_ffi_binding.kernels.apply_token_bitmask_inplace_cpu", [](int64_t logits_ptr, ffi::Array logits_shape, ffi::Array logits_strides, int64_t bitmask_ptr, ffi::Array bitmask_shape, ffi::Array bitmask_strides, int64_t vocab_size, ffi::AnyView indices, ffi::String logit_type) { Kernels_ApplyTokenBitmaskInplaceCPU( static_cast(logits_ptr), {logits_shape[0], logits_shape[1]}, {logits_strides[0], logits_strides[1]}, static_cast(bitmask_ptr), {bitmask_shape[0], bitmask_shape[1]}, {bitmask_strides[0], bitmask_strides[1]}, static_cast(vocab_size), OptionalIntVectorFromView(indices), logit_type ); } ) .def( "xgrammar.tvm_ffi_binding.config.set_max_recursion_depth", [](int64_t depth) { SetMaxRecursionDepth(static_cast(depth)); } ) .def( "xgrammar.tvm_ffi_binding.config.get_max_recursion_depth", []() { return static_cast(GetMaxRecursionDepth()); } ) .def( "xgrammar.tvm_ffi_binding.config.get_serialization_version", []() { return ffi::String(GetSerializationVersion()); } ) .def( "xgrammar.tvm_ffi_binding.testing._traverse_draft_tree", [](ffi::AnyView retrieve_next_token, ffi::AnyView retrieve_next_sibling, ffi::AnyView draft_tokens, O matcher_ref, ffi::AnyView bitmask, ffi::AnyView time_threshold_opt) { XGRAMMAR_FFI_TRY_BEGIN(); DLTensor* retrieve_next_token_ptr = retrieve_next_token.cast(); DLTensor* retrieve_next_sibling_ptr = retrieve_next_sibling.cast(); DLTensor* draft_tokens_ptr = draft_tokens.cast(); DLTensor* bitmask_ptr = bitmask.cast(); double time_threshold = time_threshold_opt == nullptr ? -1.0 : time_threshold_opt.cast(); GrammarMatcher& matcher = const_cast(matcher_ref.as()->value); return matcher.TraverseDraftTree( retrieve_next_token_ptr, retrieve_next_sibling_ptr, draft_tokens_ptr, bitmask_ptr, time_threshold ); XGRAMMAR_FFI_TRY_END(); } ); } } // namespace xgrammar xgrammar-0.2.3/docs/000077500000000000000000000000001521764210300142635ustar00rootroot00000000000000xgrammar-0.2.3/docs/.gitignore000066400000000000000000000000241521764210300162470ustar00rootroot00000000000000_build/ _generated/ xgrammar-0.2.3/docs/Makefile000066400000000000000000000011771521764210300157310ustar00rootroot00000000000000# Minimal makefile for Sphinx documentation # # You can set these variables from the command line, and also # from the environment for the first two. SPHINXOPTS ?= SPHINXBUILD ?= python3 -m sphinx SOURCEDIR = . BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) xgrammar-0.2.3/docs/README.md000066400000000000000000000002441521764210300155420ustar00rootroot00000000000000# XGrammar Documentation The documentation was built upon [Sphinx](https://www.sphinx-doc.org/en/master/). See XGrammar docs for help to build the documentation. xgrammar-0.2.3/docs/_static/000077500000000000000000000000001521764210300157115ustar00rootroot00000000000000xgrammar-0.2.3/docs/_static/css/000077500000000000000000000000001521764210300165015ustar00rootroot00000000000000xgrammar-0.2.3/docs/_static/css/fix_text_selection.css000066400000000000000000000012531521764210300231130ustar00rootroot00000000000000/* Fix any overlay issues in tlcpack theme that might prevent text selection */ .wy-grid-for-nav { position: relative !important; } /* Make header always fixed/floating at the top */ .header { position: fixed !important; top: 0 !important; left: 0 !important; width: 100% !important; z-index: 999 !important; box-shadow: 0 2px 8px rgba(0, 0, 0, 0.15) !important; } .wy-nav-side .wy-side-scroll { padding-top: 25px !important; } .wy-nav-side.fixed { top: 0 !important; padding-top: 54px !important; padding-bottom: 0 !important; } /* Adjust content wrapper for fixed header */ .wy-nav-content-wrap { padding-top: 75px !important; } xgrammar-0.2.3/docs/_static/img/000077500000000000000000000000001521764210300164655ustar00rootroot00000000000000xgrammar-0.2.3/docs/_static/img/logo.png000066400000000000000000005336751521764210300201560ustar00rootroot00000000000000PNG  IHDR9dv pHYsѲtEXtSoftwarewww.inkscape.org< IDATxku]ߘ9W탏mO`*(4I4ҀAMPDQoi#"cZ"LP Ԧ9g_?kεe=Zk9Xk ??~g^E,ًӔ>LI.$=d Ϟi{&[v)>#鳒dqd鷕GdCxp [W̏sɦIIjID̚.obڕ`v=ڝgh_עݕR8`\|dvհQA)qS%9>kzw69p}K}\z;;js>yy`nII8'{K-  >7X(/#&Lq'߉Sgq>w=^b>Oܸx]/|S@E 5Oӿdo.56NqU8Ɂkg{˧8o|VW$=ػ?iI+oySIzZy@4wKN~"_s?/'"g%NI2ř)gu}B{a3>OXd+)'O&>zbyLqNb۰Oqޞ|s88;Y|M8_rK~?.ou/{]z۾,ӗW)R'EXv_i2YݤɭxLqɤ_T/xp!r`G^ie7$M,=I8yěѮ~Vþ<1oaO0O!LEѻ߽D̼ѷb~\ɾd: Qb2Ź?fs8y-rn{?7´} Ox5I/Y?s+l ¹p=f}W8k l9=8ËmMq^=8oٚvK~#75_s1Iț}Oy)}[SC3b߉S~V<)%Kozc_7?pOzo~xg?%鵒ƙEvkZ|J YȺ5x-dݘ9Y ;V'}\NGVaS?a_+ߍ|V'ESׂp{g'y'4 p;oynʷ6~FS+'LqP)Z OLq>p)[Kso3_mB xRgzK,31ߊ)[Q9y>kS=>^'y/־z%mظ.Bn~_}ӷ>|Fp$Sb/Ϲ"$ [SpcyDz0]V1Uʹ&瘒L6ĔʹVGJRRy=rdmII6{NuT;Me7Ȥ>d$)M~HYR*JVIrz4빓,z~ijk)yè%Y9՟Gw^7SKIitQ>wj{KmoS&Mm)4vMRJS;Ǖ?>9=  ͅj5y-ߚ&/vkwpGq7׿3/O)SO O\bU3)Yc<ڵV)J\~yR=7é^ 䩿\b⩝kMVcf?WC,\K 'gN"⩄'z4M%,&ԃɟK4XzzLɇkJc%^I?J<%MEg ")=}ZO=}7=?"p<_SM3OLV-_#1pW!ɾgl,')ke8o'Չ-t%0iS.D)s4_Yq-ela2aK56~֒&?sݿ vnlԧOA%t6Mi*G1rԟkJ-tnbLm [x}0tN5t4UWkBiB7w8t1t.5D38]W9Zߒt(\>9:y/֧IaG/KR)~ <TfBٔjlTAH7X f-V"aD5D婤8H9i\ur />56Y=V?4C:D!t:09G)ME_#iATuZl[_\/>v4{/oOCEO{; 3؋ɞ>gy/9y+pLPnS\O`)&˵=ͭSbe}bP]_S&;s5|hr=tMuJ拡sjӠ:#!|I)أ~,z-t{WT9c|QbQ] _5}c=>eSWc>3~p7votBekk1MVI!lKsZlB6I~iJe+T-V叭~,,^5)?bdfR)k5l2Nf,^dYeu=fYʾ~}\^KX.Dzd,O)R.?zL.ó29NY$FIJfRݦ3_S*6&<X>۽:\MR}M˓}{MuӁS4]3}ֵ?esJTC9ŹpWua41>ޚ9oMq^/ +j(qg%=kvu-AsK}zw|ǟ熽??kඛn6|ao6(rkgXzle,ۦ\me5?+Kv֏k*+YƵYu!Ep\R[ۘn$k\_Y;fmsY y\^3e.+eSe9+=\CS=k*/M\zFίmN.K+f)dzRvv=r_w,e)MzڿO=>8/??)ͳ\q fc3W;,魒n69D6}5IJS|OtnLzNY=4T'&˔d}s&G&:Oz^=Վt89)u)}ϧ"^[F0>Yz }MtN>m:Mn1>]9MW>ѹLr>ܯ_&:)eDp?I.$꽉ΓŴ2yDt1LwWne b<;w,0?+VD‡>O Iߞ19oǗong5r1VNebr6Y)ss|K{SO')z?ڇRB)G>=:U-~:Z<}O_wJ=~kR M)=y=u5~v'Y4y{:O-d&=RoN&}ث?pLۧp{{%=_ 㖭) cs-+e.8cehs81+jE\؝ZYL2+/=v}V^k[\a.kOdð$\䲾Tz#Kރ[x߳l9ϳn9Lt}OSޛܧ4:Otlju*sӠnGݞl&:{u=UB.~5;7A}㟹זo[IW mĿ4)ǎ<ƗMqVj3Ƽx':Nw{Ku_D^nӝDe09ISR*Yv>Y?.$jlu2:esxMV<%MӅ|_Nd֔D0!"&FmԦAϚDgIuT':>NP˱6):M} sӠ/,M&ANNS,ݦD0 SM?O?_f- W{/>~X 8kcX#^z )Â椖-[Y꣝͓2NeᲙ)ij?oۺ>}9LVӘceMk }jO)n:ŸLRDg_?ڛ5>OV6':gT&:dfI%$8r^I>]9Nt6>9Ltn煉ersy,=L^&:Dg3Y}S+צzsO<7F 8[eo}%}oMq޸m[S7n}p4ZP[\i}lCA[J5ҭ6MR=ti%k%k ½|;Y%Nf:&IYSpx}--mP\BgFy CcCg?> 'NR:Ϧ29q tnSC:ԋ9s |P8 ef}xQ,DW$|5[8G޷1)έD<<.ӝKB&&'_G*w0s[l-';gcȼ:[;fSk\OJ0K=n!Ǽ-ǽ =z#k!tkCBgg j졳ι{Qt d&Bg<6 /~`{)55 p݈giwJO)?yjqsIBC穆Cg%/MOBgk)%Եc:={ vr\йG=T mBR9fu {:ocݧ8,bKs/:ฎrN:>}r%t-Dѻav=Cr5ޘ6Ԏ_', +"r}74y[1y OqC3{\1K} s =`N56s6}s o u tﷇ sꞗBg3dmBzVkIlOS=th;LcsQyܞ.s }-m%tm Bg:Q3oVȒٵɾql\/"gYy+%{joi1إ8?CS5%tD&:{< %%K5NmYmBgy۞a~m?8ζ:c-jriϖ˹1tnѲG:йLc!Ra òR^ '>Kx-c_ɡgw3z[ gepSW+i#^v_C”fsNi.}j<ξMJ}r+,B:caOTz>:yv<{A6Bg^Bg Sqlss}d>!nH#Vm#dI)Qr>L|Ϊ׎Aqz!NCg!tޭιζ:[׃ox4\ {U>Db_;ً5[׮]&:yOq^)Ωes-tml:5RLmmg 1o Ԧ1mZ]F-ԩ69:gйlrWjqzcܯI::qsu<pCgU:{ӟkܦE:Orqr99λSSW|k5\ϲןp\3l~$}Ie?=B=}i^S=jzԧ '3/Nt'1t.Ó,LN}qkm)eγuT'&OtnsBgߪG4*Cg BgSRvEVe0k/{`l}Y%*C$mBA㓖1ZᑱBAζ:k!tй{ =x\ù%tNO7x4?MK#r)x uLqrOqg[Dg8mR-cluZߤNt^ {&;Cgb:'_Ey!t:IyZnw6fsK0mi!t-:P85J\N[|(tA9!xn=:i%t΋_ڃk,I0fm\"gٰ_u䙒hss|+C\k1&Nk>a)Iڴ1tHU-~(Z%Y乆}{Bgk|mIC<βOenәgsjӦB~ ӟc\pkՐu ,tޕa!t=tC<=._C$Z>r}Oq^^S}:[seS/}brdf~ywS[i1r/ ӾV=B&:Pf sI1BgkF:4yi:J8\>%/8D1tQy}fsމ#/ pYDčo4]zs\:+54flj}bsVܷBB8ٷ3{s\~b%:c<vߺdv}F%tXX@-㶡Yl-7[ õ1Vn=sOйN0y9{ },ns3mzv|˾_ݿU9΂MCx+8|}o}ӧ%t5:X8%OKkcj4YybW9TՖ[Rlajs9?!Xѳ':֦>רy0z,Snsι\cΩ$:h~OCgcs:<p3f1uLq> 1S'w8echjGCgeZP&:йBqRN}\5ky&Iy MeZrݗ|)\yCz_e`ݢCsxޮ Y.:r C<^l)Yck':-t!t箅x1t+:x%dM[SggSo/ rNqGIS/"b}ʥ+ąCۦ12j M5d.tDgk+ s59BgTCgյԧ=|TC<~BHjsC\/c܂:٧I6y =Ps :s{l5t-P^ =L/|@7^)s gdgұSAOq^v)SS^ܚI:DgBgomO}V/`>Nй8ѹ,څйWs {<}:t{mBj9:a}ٯ q kP3yS}M` RC< =}C!jm-t6YJL}㹟5zw{q\3,${7::S1tV,y CgK|pr{zBgCgs?ΫWIZ dй=1>硳z&>:kxn'K =(::8cr@WBg[ cpw1tI;OYs<_6 YgLl2<DT?y:~1t.׍DgK5bB&K6YY>滭:claʳOnܔgsnsB羟0Dj\cdY5HOzpy1tD\sʦ&::_<釾o偿?$gpjz{tx~.SgX>orY':T]j>Z6Y!t!t.?LKh3zj@ܢmӑg6ѹ+մ1t܏gχك:[ }KLˮs-S[[dqt}֬ѳOz@s=BD[졳BT܂oWE 8[o=^!b>)[^zs {NSY]:J6my&:Y,=~A+%5`-twjS& r[1y/<>}S6wckIdendJM9jqr-˦jաϋsi[5-*OQnpX2>c)os[,וG s=d^ s rv}snZZcY-t:Fέ}7K#r[1y=݈.=9ۚ|zF uZX\:TF;} s<Bg,~ms SMCd)k<N,tN1v&*z,Bg롳ʼnΡdn=VD3r =Ny=..oy:pչ]:gi!Jn\,Bl5\Dh:Չ-tná·BױfJyL_|r\?"g(m/7Ӧ8-eqyh؟+LtGBC:[\M|_ȏ)1T0Xxُs=N=tAe\ 4f!tClC~c\&9kc.wgۏp>|e:y{]gcIr åfIbyn ]lja tUc)/%2&:GB9=ZLFyX=W ,u1t G9{\-M[Ndg+v14 IDATΚy 56aYl:~g'}'/}.3L-ĉ[SN}=!lxg^^F s6Kͪl^.qD:2z g!R.?mdžqjιϯu/idIV^˾~ ׆كYcXfsHY'AY.+#Y<S ׾~WBpN Ozp-{,;K1tsJJ5s9Ϊ>m9 i+1&eJYNqM:B09 :^oµ)'wO\D~vTcd:ξϝ5UCgOtoc^x^Kot ey SGu!a³ivC쓟s{o{OYy)gY?B约jDB)ygf!N)Sѣqµ5^rJhm{χ^йD܂:k R}2sK@Czz^fV#< {u:tk,MOB@ { K0\k1'<'y4qt=ss&7H9gep\nks|r D3׮SOp)+68Oqp|>x:Qɦ:^)8Mt:h^JJf5s9X309ׯBzFlpmm1zd\J쁰BU˧!׈|[)Clu,YsCg 5q tӢ}j9Sn "gYdNqGW8p4Dg)%Bghz =`YYj&:4 :{l:%K5D,t=t-q}ju f=.s&F}Rs6SR m D`\E1Lq8+D3q8SWC#{)s|CSK|D:8>uy'#V̳&uuS>yAX-ږB mfa:Byp\sC-ys!t!t:[VΪrOoB?\gcY!N9⡳نDԹOwq8}e9Ό͢ڻnjy}es:D^ǖRD?ɤ\Mu:[^S%czl}{{,B^ZS{\em6}&29؟γm*s s m:װ96iι./=Me9">wkݹLq޺n<CZ/9dS m:skBa㡇YZamGI'0kBsZ~N\%)I}#6k\d;:'lVsCB~~C(9Έ1yK)'׷zpOtV\2yXny%Xd)Omb>9KCtڔ:[ 5D}:s G>g׮M^.s,Fuݶ1<>Y+s9كj\_uOs -Bpq\⼷sB5W[4{h>?Y=6aM+q Ω?s/S|D3aWz+D9y˄קOqcxk/֭\й%UlY%tnzCgx695")̖5JVB\>3}fW':k -D-hCs -mbsC]Cgks朳7gۆpOcA1إ8ϣSbaZQqkcKْr '%IYJY9йFĵ.r $|:{wr>zCDhSf_+D˾G8й=WX{+61Κ>C\眗B/SOBF 8GO\] d78_>n1)k/K)k'⼿ay:[9]Ou1k %jqr$z PC~]=k d2:!t8~s 9n6"g9jףǍx~j\f?>nM=澷dc>29wAy}?g>37fᖁ! $SBV˜ fECxXct.On>o9%t(tV+&tV)й]tuhslgcBgQouf;Xq'c}ex!];9EKZx}7z>E|~'^g%4c|йY~Ρs}1A&Cyci..܆ݢs(+ K lBg59Nmeaz.tNEЅΡ =4?]q&8"g Vg޽,8{ɟْzl4ĢnZMl%t'K7{sD#]s^^C<Շ^Bg)1%]ϞaS:9-;H :ef C9Gv\q^Kp'Z+ΫGˊ}g_A+-L0 C|5RI\t!s C,ɼ.:Մ. )ΏBg57AOoй[כwGΡs(tb*'lG kVcr\tLmʵk<ʺBeE9yR?3Dq\cysw5Vט=&V{Ңs )B[:[YtVsGs~iqB8ݼWCX%tV =xk 5V M,/aq:׭Jss\^t 9teZiA:_Jss%kaߒ7'xOkDXq^zey]WskM܅iY}n:{scK6d1?Q\CgZבSX%C2K5t*ss$??:5zzsws\;GD 8A㸸wr!&;ƊN}OqŹ;Yt%h]pn6(:΃Rl:h=1[x> Sdl1vkBz>й}^Vs lCgbɜBg߀t<6/Oכu::䲐 Ab]0>xpӈ'5VޅsotygbŹ}܄2|bYS7EA0YMܾ6t^ r DA*+!=^3CA9RެT:Ίr=/2[y^ ?!u:wAts]cਈ'f9jdy[q޽5L?9&)4Ζ"&t8ߪluAй6i}9t5cJh?8ǸSjJs|:t!sX$!^;(BCD7s}#G?["r仹܇{}W^c:df1&B>בL}O syYݛב.ՇR 9LB~9TpkCr_S^C0{:41^xΡsg KmΡ)tgD 8!QQWw>+{޷o5O=^\]>w=֞ YM`܆!>tWkCtٴ]CgoR'+Gքm,GWy ^֐MyQrsYxv\jw+jWgCgMyg8IDrBSBuy> >l8/-T~yR)t`CTzVs.+ssњjkCP_gӋ9/_&C]C0[ s[Ke뱓94s]xft9>v.j6=$>gu/g{8/Y8+k̞;3kZtٚn^>:.?iQn2:KeYh/C;CgsYgnC\?Ay977s:.8йяgp:C9Ks]Zq^zeyos^z6FGR:+_8ӄΥwvinѹ?wsV8_KsІ3?Uu9N+>Y^q.KD,йYt,:BYqN/&ǽ?u3/ M9E|Nܻ:==SV--MBzLk:%P{ 9n3)3L,81{SZok9ȣй2{\ЄtͲ\g: DNJZ{'W3sbz[5okR¹ul?,]K\:q6CTC&mbtn9rγӚr CY.?':sfC^v5Fsʜ1: i:?s ͯ69N!ͯ8o/ޣ _9\cw>ܥk\5O1X>s ]bHl9xαky^rnrdq&\t sוE\%BH zs 1<9.s KH/7o9ksΡ{ pBYnGsa+3vZq^2Oc}y}obytQ謲:hהB2r{ sQ5tNqtGR,s:7K)J[}nxj9}\B=Xq8D]lֻ-EOgqyZx8Oc-wrj1tx9塹Rpl9%vf:ΡrjkS`:7%t^5&֖i 1hCxNIAv c:{:/<Ǐ F 8{8OGƵ8o]cا|Ź=,Ρ S\CnѹW5UCgEvA9/<:ӷ܆ήtsh^GRsHs^MйYt. c7ʊ3"r<=.ˊıXq8~jŹU]tVa"tzr++9ơYtE,T":˛E:sJ\.bNz:sk,)Vrs\Byy:{^auHd~^]w"rA1ׅx+;ϲ&W8>fu.1nq )l9UEg+s|% $Bgus~ænYv.s]e^5:{,8x;8 3pPu FWgeyYqo..dm:[~Orp\B%tRr\|>WP=J]ctn>z R:56 b s輽ڜ"ty_&"g;s>>+k.]\q^{K+ۡ&tFs?kpKsseclA D\9F}E+ʃrF!]蜂@7s u SxC8s8p>w3A 8A{k+γQ"QA+{޷vo5O=^]q?w5Bs_G>L^Z:Eg-:cf_v9>.9tNArp8}x|nUB||[zn8܄5t\!SW=ӆR: 9n3ČQzhtd[YbgG ·,+Kv ok~+γϨ_tV<[56(tKC%[FY>ƞ5J,B{89tVe~9³TC'Cgמ|8UǶrk;ocdܵ0xʊw˛[ǡsy7޽5ۗz"tNs ʁs:+C9;h\SeC:t^~ˊ39Nxxs!#+ά8/|,:7+9N ]E 9Lu9rTކzs(ksP]C,:E,֞MX:_/YM>t,;~&:q<n 3)1c8ˊs:5rls\.Y^tn:_tHDŽ1B*55s$)ą tk\9}o]ߙk;p"B8cOeyأ8WUw7|sw Pf6~s^tnI:(ښY VǡsirƫބZ1tM,MAa2nB筟}f"r<v_&fyjĊڹ{WV%w_ٶ/[b&LvAuǡkhb89=r܅R~Bv8n*+pzVފNoŹOλ,:Bg[|޺9>Wc Ex'w*2kCڬk >CgW:͞ *3yVN3uy1 8c|8~veC=UC^1r:spk:7+˲~]\Vmzѹ|&t:}<%.sC|:ǯ plDr*+kcy-]8iYq>$>֊s{n  y{ە wΡ,?Ƿ&BgmޞkEބq^./D}lry1sa72E:/n3⼋gyŹ+0We9rˋk1tNsq:{E)Q8Vjtr<]Vki^c>{Yq8DϊZGļ̎.}fV^q^r+kޱɺAf+m輽fr-: йކ%tn>¼:u C;|Z,(}axWup'c:H/N8/+γcV'}+㧂[g91(5 }2t|XדڜB܇sF C:{:.jxl pJ'fq}y܁xs\=Ɗ5|jMf> s4跆0 Zr)k&txo\gsl9N_+ S!ΏsHsoiy_pžs.8^<~sV3LCgxPYtT"e6tG^]qr<}\Q#r|&Bgmkjbd9N+_q>$~W#8k}:csڈ8]j̬c<vY6zv}kϱv^uΡsh֟CZv̧~KXq8DŊ=8wtcYq~b+3ܞ5tY%tNs::{:ZsxLH=W[uٹC:˻95tnbhs|-8'Mû77^gEuFvhs~͟}| ʊZԻp.+cyG\q}XqnύsˢGæ:_N?Y.3{J<.Vs!z|Z,|1u|~=2k(Ad.τgz*#i+?uk\z-&)I+S0I ._yC&KpVgw ϲ1ׅx+7>,8ϸ+γn ϡr7%K_EfѲ -9fWsJUd5ͱ1|t ^WcY a!~ & AKy_p߰DZ8S9>pvMnFr'I"ږE?s^]Ga.\ϸ75/788Ϝˊ͟i(ΞBzljϫ5e3sr9̬ MzUHKӲK~yb.9YJs]z pwo|}u~;% 8ZƖ % &e)'c>f?[SkǷXŊ=8wtc3wb>絟̊N}o|TSQ=˔C5E2\CgLzHq^t.֟I[oωjN˂U>dC KW#x8?{r? 6he&}گ|y,:9dC Kl{ Ac pj| ϝ=Cwtp/p=~Wo`̛+Law^kyrE\{<Cs_Go y ,:ל?90ykѹ.9X9+6]rOY|6tNǩ $ɬsˆaΊ3) }-{#ُJk8/wJSx&?Wz"`ʁQdϊ>qLSvZڕn"T=gu/g{8/Y[q^OeŹ{+qX֓78tN!qtsMm\zo\Oar:M o^g_FӼ¹wO;p|>ů?Q?q%ݛ?V5ݘI>_y~<MwZq^8uE tۡTcgIqmb,Iq3+λx/c.YxIw;Xq ??lO>@+SYq98,8NJs7 9tJq Gn%)[~YVנ]!b G*J߫I88 ioz.I_?Vg;wqewox'_;fWSYq>9ySE{\zG]q^~A玮qW=Ɗbh8ǏƒyzyP)NE NkMlr;w翈 _ {_L^ 9n1n=g_q;=a+v`5<\gMgHvߥsBZ- R7\&=kױkcy\{0{Kϼ#{x#8ˊQV W 8IΙYi/ 47藃>sݍ\oJ|%]z/I\gӲWx͊suk+>xߥl Bg+?й+UfY=!DB8E:BK[-:yo^˭j3*i؎n  :t5OYUNz93oih,R4s>wHvL}g7k/k^nf rُ>un=;nONJǮE >}+vIf-80 KlO;yY5t,+9f{/+& joj[Kߌ: 6~p[?a:ܰs&# yك+ R},P1`~%`j蜏qzugz]g9J~o\/~',D7/K[_;] }!+{,oww->/+iP{zyy律HB c#fsz,:Y)hNYcA},-).K!䜧sCmM px v]՛ƆAg^0Xl^B< M9HDs cO|,x+֯3/x2pnc\_{-Rpirt:? p 7y`rb IDAT Yqvٹ5Wm` c༑L⒳o>˯L~+Wظ1>e_JRˠp .].c/$ KgK_ݯѻy{S6_{Ź56?d[U1d>swb{,;x8/Y[q>]݃yXo%gS=E1DxDڃS˝%xft|~>7.;Bv8?`z0U7k8Av?h6Kf Riyy]Q謭YqȡUS|i\(k~:!!q]Io?|Źk<|q2iS/_t:+.>7sqs4C|j$}f8G_/q!xJ볬8VW 67Ǻa2tcdOs1dvEЅrN DԇϬ8tTte5|T S},gW.t9GK<׻k3oﰫ{'=̊^f|S\-49-1̛E:7Zq9FˡsްYtV:^{:{Yҷ|8>q,qU Z?ptӅΒPrɂ٥:{a| A1dnWK::9t2/R9kuQM}>;:y^ͯ~_|qA8q8OZ[q^zey 8ǖйm,:9MlcHsKf!bzm&t _~' p79?"関˭sY қQA)tbsb]Cg ͕7_1_ˢ_(t>t~ڬq}'^ D7Xc.L8o] 8ϼW{+γS\X:HY:׳Y35M> Ugo1 |w}:(r~efWn+ɡs eR,!լ:M,y>tޘ {7o_N_q9繛߁s؊z<}}LM'2 [t]謴݇g?9tvY E߅ | w֯\sy1i.AnJ} P::e1Х!α_NPXLn+caР_Gik,־!|x5}/oMok(EggL.߫Ww=絟Ҋn1}g^39i^>?G-Ȗel.J2*I,yNr7 9`L%`r]pgjWMtLtsZdtM,O 1!,kLξQ\aB+Ki*<|e\~i ӢsYlp7+.:% Wx cEͅ˾}4&Jm}߻p00]q]\1^tvyby#ic:o$߸JҕRCgK5ttePP2ΏCBgcח% ݎ+\3?Ͻ}w=B'BM8<z}z~yoy律++򼯜ϊohi}O>vmK~Q ֜soKz{^.-R?/_CՃ݊o>T_㐘xwwz\V9͟;ʢ7ɗ?yϋΡ;gyf:g׼3 1F?lݱ7oכSØ*)vOqsI)CgOT.tVk,O39Ϗ%^C#6\>q>_x; '`SWW.{VX{6+·O3S=įY͢s{GwiC:ᒧ:啳 prV#ߥuw_}+ɞbAy/J L] )Iѳ6RmWk)r2keTǷ9(\H=t}z Y*vZﭾpg]yq&&޺7s6&>x}]qqk;5ncŹKe뢳d],տ&d/٭=y5|/c1rVopIoscYd%tܬ 9$ml9FW,<65oM: ͅ!:?ЄAvO_/-w #/W~2AXq.Ǟʊ̱G]q^{f/8{k8wˇCg(t߹9DйF͡#̟=_̊3/=y$(-ݸImfzlQ\Ǡܘ|yAQ :5/,rŪ<4ȵIϐ *׉_37W{GNJ=M+O+n7/>絵ü,9u<ݤq0}B\nq)-9F +O.]•͕u+% )\.\~c_c-:6]qWӽ738/wW-+λcy݇]c#8s,wihRg~_`S}ey}]Zq&xm /_}r+~VBn%p )HgHQR sW1|Ρ9=Ρ$T /=W)^ M % R S"8hu͗bl~`08Ak+΋OrPsG`YIZԻ˩8k3[~M8yZps͡7{`ΡnqAtb3397a2r97wxy[RÚXBgƤ֜\%t +{ZvY\k1tV:?sxz?@/:?8Q7v.+GYq>$~+73Y[Ce)nVҚEgoc܅3Fpk&#g潒|a&Pw|s[ aTBg /:+~nSl1zv ,Λ6t4e:!Ηmם7|Mo&=~Vv~@wN\q+fŹ+λonZ]4ocܛs^tΡs>Gl4[y9BcLGf߷|>+;>k؞|?qYaK Ai٥M%VV?AaT載}MK:? a:£/=כ=Lc|Ͼ| wLxͮ81{ZԻp\q^ V<3Y_>k\wgrWй,92.ikѹWBgog5獺ysbp Js*_ކ)|~(;7x:RZq^ R¹u{Pu/IYjOwsGwjYV9Cv;VND;]#P?)sl$Bp):o$|g6tKM7_2>.CBZt:Džg%t)tǞo{~y~$7{\F8EKCWpw;wR _q^r+kc+1bN;ff0HKOL&xCrwcά8.r~P|a5r+͇4&_+Φ"6tnk7{Zu&tvҤ+qKWV>W/v?6tG1t~y Zq~^ɯ;br7^/Ts<Oc}y-8|.+SƹmŹ;,:8[zB)t.YowͅYq#/hϸ~W<Bl`hRm 5gQ5 Oxfb\ R,cy~>$ WH7d $gIA}^7>d"J?WZxv[ڕ 7w^q^ڊsa>'8ޗ=sB >l+<5^\C즧nŹ?e2/_LbLSz6阍Egmכ7q9.:u竴| WqYRך/s\y.\{Zvv~^oΡJQWo;+Κ5?-+·ׇ8EgfYz`(+>uZqav89>]*1~+ߛDx紵8Źu-rA8wltV ZeʣAo[ @B3\QJM&0qs8mho :{AQK5o :GVй+\qr9l :{+J+p-ʗt>nEےUy/tP+xMsziq $Z;UVYka{]AP\ 0!g#Ĭ]]{sE!CaE03iqLwsNJ& :7сկP9K`T!g.[;C5 :Etzլ65I+.EaVйlxl WBr=ȼ7׃KFZAP2mڔ&0L AsB[cU㎰9\MR^P&ڶ[f˴t5jf9qj!Z1}Έ-Ν7ԨeICSDf !gYl^/ %vl-LҊ+6)}BRg7CzfeJ՞҄8w찞G]r3ɣX&as Dgsi:9Jkt{nQfo泭$UYLg\NE0瘵8| Z=f0qe]e=Cz_xyk~G֗涉3 {la$(1`sb('ym aM0Ljn ,^C.Y=lĎPsYTp%P`Y F9T kv6#A1BI[O/)d ?WXZn N[ 7{L@ Q~3oVYAfs%dyȃ+)I)q3S1s7&dnqiikfzZw27ts5o!@z@GZf& Ճ ͣpGK@R(U¢MMR(y\^SV$_ilW,˗pKR$r}%/%)\ ˒!GHE IDAT 1ϗYgĜ(^j#pl_-9;` I-k?wZǷ8S|Iz-]w+W]t6spn˽kB7i!vE&1c-5Fgt6I,ע}:iqu\Z{]6tn5:6A8ϳe {dgc[2pLܣMѹУM y({oФ`{ڂ5Q{s#Me͟5 Gh(<5HKwS>g$,;[B k7Js?B[c{n}lt9SorzKADwOZה+<#1yñkqn߶l237:[AgST5&\Zf8bբ&(rظZZVlz $CfӦْ[۞@A6mܽصc-W﬏]紵 -Ν_gEzճ'kqnxXkSZQ5(< 1A3Q(%YXO^ot :[ ۫=lȭVEe5AgM%E2Ԏ(_䌞:jqٖ礃✶硶8Ρ޶F:=njFmwj-Ϋ֣lqYѹ-rMX_۷v%֊Kk*Tb i)6[=ݾET]j8KSy-α*a--=Ąs'Ź}m9)u n2nym׼Yi'olvW~IaQRXo0trw5E},Iۮpţˮ,jK.-K6',3]ejs[W3N`?3Z5$8w#vܮpn\%jt..|-䄚0%igfmqnZ/-]pMkCMt#8m%oW'P<.zi`Um얽9iWi-IisZZȳ'-Ik9)I'%maZzl9 07:Kx9BCId ֈ7{PLA' $'{Ӳ8,ڽ6sW1za[Xۜ#embcpZZ{uB>87(ߛcÔIK}@m?tchVvLmA@xѹ@ZFZ]At=,˞}fZ+ޯ28'#[HZskZ%@\:d]Ak~ܢsѹ1oZ۷hktnijxm۴.?1iqn9C({P]紵hqN ✰Zv$?y؝9v 1g8Uv-ܴ !vN.mjew&SUfIݕ%mz:l,C2 mIC.鐼Rx%/{GuI0eH:3~Zksh1}hs=,k6[VS">&3v,㿅 78kM[ ^T+ |\L]-zjQtwָ$Ϟ(5,ole2ܾ,(,}%jwL)c(PF<ҐC7(cnR@rs=l%ΈV|Um~^3yLZ{Ź/(Zgt-9 !6H D7{'uV+Ͳyr=Ln4/7Bnw-Us׭yww0vNKjb^/-iF0èy]&SC7y負`偢g5AVY&i\;B3n8.9\:[c$8egLjqN{&͜cm!C9.#Hvټp Yree$Ug 3.Oy}w|mL_-ńύH%]APoeʟAFFع7εcfL) b[8XMLIk!~&lqZ| {Nmۢ6L/$V>[޴H{|ߑL)Q^oqt67cn-><$z|٥ AGAz{sۃʑZ92`-)KHJ~mH;^ir7munIVw\(؜L~BRWf+ _pӁkl?|!JKpPosyZ3}B7(ńnj;YG9փzY^:[g$3j^;%/Ic~MZsiq6Z~'I3x.{Lz:d}ki!ׯݤ{k@_f^X ./w-tj0&$=\O_lerW>j7;QebM$=cK[;@hy?mqugsP?a*M ڃ΍CY y=gApq8ӣ`\ 7:ω]ܴjsOjK缦N9-J~v| ߐc֘yA!OrYY&M Xs/v$]Jg7VXJ*SjُU [X7|ߔ$ok(8{ݢFf9(mRzs\j3g`t\#gCpGrv_ނ;gf| n9:kqN9nbsڮ8e&YR`W.0׫erf\_$X_f+ /v7=$r释3wsBg%VLK6 J^C){$I >BC7;E.n)KK% B5:$j}qiq0y~gf_kkqy\Z縃lwcolEG~C]E1Lg,zw_]/z G!1\p\y¾mwɞ+r=V_ >V-cgK\h#R~,;F-1F8'IkqNzMkqNZLc$Os\PfZMz^Cܸ|LRsLp7ѩpԢ@Ka)xEC`c[\eៗ5u]#ӂRѳa"ϙ]̕fGe9fl*z0x $i֛%l!җFzйjn:P\ǥꍽqZ ^svõ8e2aL2;s,:v?/E1O,z3ezesL$7=u=ֿG۪Vɗ-iٰnTjy볕?ٱmB3ڊUIZpm8g7΍Fl2w{WٚA Zlp 6LmZv\Zcypkc✶v-8K:8W;ԝ8arIGcM]SVk%m)z ǃ=a˛fJ\\X"ea-/av3]P4nZj>u :͠\ma=l By-5-6 65nFsZ3^P&ib /.u)Beo(zfhrsZ$]V}Gl§kax̯蹰a=\WTlyaNK*E(V3SKk✭Iҍܽ{tvϡGfй|{ř3w*{CL8>g(-έ,ԝpX2Zzٛ/zI~os`}yrf_o'\L(&ӂD7*cE(F3|ͷռ-vo;-k]rsmAgIڂQR\5-6sqs̶#mqgf_k}Į Osb~'v>c% .Q_(zje/@+%;Hefsf+ OIۊ 򗇡2[Yy҇9墇NqϔH(l#͠Knrww rͶnzQ89-]9M^v^;J{G _f;xr9&۫wW.810>P~9&#90ffwV~Z+#Ex??ZsەʞS:BfaWy89x5GCmAzs(JLׂ`L?]zs?B[Z~'B fnQ^v񀭜xsa St9&.zQR9&зK7!0=k%jI/z& ݒ&wN/쭖+z >kYŹHCc켾]Q9.Mhږ;wU[%{8ws1^1aUH_7}mgӟpa 9oE1LzqEρV.zI/,zIdz3 ~xP^;z|vzs,WXV7FR_w`0qnZ54W@Z7AgÞK')-`N+*qsPdw8-i'9ŹHnqNwelomstWkscc,z ff&ROLZ/EV+ﮅ-&Hõ'S-:BũG Ǫyy孒|t&ȁ1-Ag7:!c cƂ3Zp 7-hDž?2]#-v r$Mߒ9AZs"qiqVoﵙ[_{7rcCrIg`Ti`KvnsLeW{Kl`2W&{r|֡ղQ,v fU>J9IAutvS?gYrkGV0Jn:}oشρPo r m9]u?iW>38'yqiqN[;gZ ^IcCG9&޸sy3Eρ۽r]]dz辛۾}Y {_]E#M˛3_BΒd濯f d[BiKǾŹ}gFsIrLٮ;5Ό}2s {}Y0qc$}>8'Ikqcm&^#y--ν1ڞ禌-2˜ 5]ډmZ壛^ҩE1J)zkfN/|܌ɶ[noxfm)z @>=CΗ|A>Ѵ8vnPJ y^Y:!gc%0?;HvE&od_l[cKs1hqo G72lխ#Sݺx//zdW>G+z c=g+ [10fT-leT@lbAdo(yuP*Ƹ8Ǭnsm,) ;:i-Y/{?i8Y")|fn~ׅ82Ikcjq~ޛ8#^-z Jc%CI4en IDATާWE3;{\yR|Y$M=0.ٟUwMϿh:1r}k=Bsc$z-i ԶmvFyW#C -#iq'x]h|cL[c9{8;1bN=koGe9U(t^aу`tf ϯw}ͮT-z&`9R-rvEEr$yvS\@my}YVzz}^qmqnZ/-ikZ۞lYRz _6=$3?0[]t3;{sh⿳bc}!0:R-$^I*z`>{gfa%cr[P紵9Z )_88w;^GWlqB:QIgZBcC Ą`$\j {ܽmǥ9Ƹ8ǚ紵I>NER;cO}Kh6GPlG򚢇m W~WLsL[8yC`mVϩ?noy ͮӊ&]z$[:[}m3jq^Ij8A}tltrǃ8sb[?3qѐޯ>ۅ3~Y~۷5dbX寐t9&K/1=6¢@napxm?!霢& L+/⮢I)|?h nbJInk;1XswИi2PK[IƧ9sFnq"$IkHlO:N6_[Clm.X~zm7Ei1=z`0kafA`F91ۃǯ;PX;-\͟JEL0sO5}<erd[|إ7YMBcijZ:XQ8JOۦwkݞUG⼆<-iA՜nj0*-δ8jH.(3F-]o3Y'SKWJ9&\@oھ}溼9&Qho^.z Gxރ͟7g`;I-1F<̾v  pAPxǍ{Sws聿ksL*^\-_tns EϕtisLcV+jC`p;-A\,(γ}n<EуF_Y\$8Zk`8yqs̶#mqIs< $^sg}皤ޜecLwR)\rѳL(3 1/ cvfwc"~U\ey5ϻt~ѳ]S̿Z=GFwY.=>{gJ:Fs1iqϻϞc! ]TlZ^#}--=Ąsiqn/y/ 4=xݿuMsLf*/,zVv\)z Go*zlqjK\X;&{lm = l%钥OJZj=:awNRs̶q-΍%9v }mil磴8g:.-m#ڮ}Į Os\|Jh?ZIw<2wa`Z'g/|ec8JB'4s?[h{? v/u>Jsƴ8\-9|_ٿ仠~~tsh|/XHn9&/W?!LyinE1+W=33ҢϔP g|33֣%ғsosH:n[B7bzCc%+]} /8vIŹ}ZKs՟c.N9&Lkyv^2{sP咎=DŽl%{CCZtp썇Zs tG7$90[&q lqpU@)}m}~)`Ym9|:R1Fۂ- 8Zp.t&o yx-Ϊ+Ctw~$.Ssũ瀴sy3^W2}aJ)p''oYDѣz&!gIzϚ<>B7Bc1 ti,28'_@OQiGxs{ysL!\efIٚ%~?E\҉\mAfŹuco~\ ;ΝyyAjsۮDk}c6Lyvڵ5/KE1j?r4sc}}1{=Hу s\zI= G?Ed͛]zd O>I Fs>;?JIg6sn8ψ=<oL5mqN?%L܍y-1ְ_]6(cBNjcBЮ)zIcv7csL$_"<OrigCfǮYU(0NFr?vJKR9 5t6>i-(%#=]ۆvmq8wWM| SsAվv#{s>f=n92L]ǬM9?ojs|d2ͭcγ<~;E{r *w ^JѳL$Es=$\)zIEρd9~R,6/ܶg^,0.¼O6ݷ+djɫaMz9-tXJ o8!紀RRp*\=W1忱 ;λ|SԐsƬ1[9oĐsss<9NZ;I;W DZs9S>V_ٞpCc&?؄@۞z).핔# m mqNF㎰91t8-Ήg279\e9ص8 82ǎi;߱rWSV^-E1Ubsx6300E= &nւ5fcjEsZh,8A霣*},.}".Š mg_9I{LL3-ν!Z[2}'޿n|P:1[Q9&ԑMZyܽGofуlT3o[6=?|Aܶ={Q#>T P 97|tga+7#t){HCci9(%1Hhͯs^+c;λzq@{Nr؋?d Ķ?ŹL$_#)<dž GĜj |~UMf)ޱ4_0f+ tYsL"}/)ẑjy)zI>Eρ]4A/" 8KrCߢ"eȹვ+$iIBBc McK;WRܹX)yo?8t^EƱ9Y3 r 8w'iUs$'oqNl~~_鞳繪IOwkoڲl խgZoY&O;|O=F3[99&?v8XnYL~bDcrnio=]M&M (.4Jms[s38z`KO9yLwH[po >(NIw.I1}}&_QOgˆlk7r$A۽6Xc$z?c?NW{.Y[l9X9ujZ9&';p>ᙝpN˥%.z S3sjK-#A`Eȹon]*-f~xe[ێ}^=$rӋh2kʢ4&G9Z=nI[t0q]ȹGv|m4._tfg vݯ?Z~ӵ|~A&Ͻ^5fv?|ￒߐAs}v@3LCE1.m=Em߾R|NҦg0=G+ztvѼ$bfGA`uȹ;OYu=Q;yնChqn_yO#X;;\N 8|LwP?tt=z7[d=9t)z t^^w'iY E/:|= reܝ+KHG*TN%JD[wۅwG~;H\Uo`|ܫagiZdWLO/2%Iբg@K^8Y?fPӇxbTvL?1t'igѳ@F5c= rGrtl;͂K~ztw{#Y*es'};,U3 -TM&9&_:Eݕ N[MN/zq 첃_ 9r)A藸]&y5t Z-a?<ydǾW,(! KB Y, eJB)[iaJYN[fL(t tZh@ J(Y(e $ۺَؖY_I|Iw;e[ukW IDATض=Nq xnʖN8`7lmJѪݦ*;>'% c:4,oak]P++z$DT/֔(DDDDDDDDDDDDDDDDDDƒs?O38@hlS:)c,#aљ ,8~ 4Sq˓k фNt?R͉[L(n(s(>b:(DwM4%%a۹e./ZұeA%yK)Z 87y\S:x% eDfMEg"kO/ (F )o@/՟䈈(K\wU]mfC^ 8ݦJf%F :M皓r,'J;zBDewBDDDDDDDDDDDDDDDDD-4~+v-l޹迣;\.+t qLgʈ#Q@ܼư!} ΃e#\CPU 5g""ʦ"y9|*Hu7"߉ew#Xp5O/e9v,8Q_ϐxt"""""""""""""""""l$"SW@@Ћ2p) 8љ9 b?+ DDDc'".> A,>Dy9A7 ќ;tJsM6'g|PΔ DDDDDDDDDDDDDDDDDDIE&Wm:D> x `9Xp6OD),8?b1[ xwc87C"ʁmKm_pL;ا7]y(_h3ÇB(͕>uD*1^c:qکw BDDDDDDDDDDDDDDDDD4V,97'WXIu}4y$8ss76dȶ]w0w4S34yR.\p-c8Ѥ۝ vܰ]tZ/8t7B53Mg!"'|]JDDDDDDDDDDDDDDDDDGTRJbq?.ݭ{oiH%L] "ʞחPޖd -9{}m{/|QΈăa[8tz9YzZ1gܜv ~&V:(VBDgjٔ|Q$gª*Uw[r5yc 6tO\ۺZ:@rNqRֱLDDLRceHٱ҉&,&Xp9^yr\ǂ3ѐ2~hU'9S䩒L/1TLqwy̓]=+vh yqٵ''Lt‚LqZg-hXX)a'vC;e v~t\jWz0̩hӔ;t8*^OD4grjm DDDDDDDDDDDDDDDDDDpR.Y>B@XH('~`)7>R "ſ\mu_ gh 9C3SeW+"`3鿘wӝ(mh:hLX/*QLqb]t}TLu=47pv@DҲoՑ({g;8D@<MυˢMȅJ{6oz}S{sX]`,DDA/bNADDDDDDDDDDDDDDDDD|J~y֖O0dq`yɮC3MqGK>`SYp&"3 z>[Zm:HvL4Çڔ~&@FOF kDd55c:奿&z"""""""""""""""""*\,9Әlaɍgr7DC϶.*| ڻdЉ2Mq|8LDDE/D0ozsc29a,/oK8ty,9з ~u:=0t;E;ܽӁ[`(ә(?Dsǒ3YO\ t4%S-Y Qx"%e3UInz Y ;!""* nyPn:Ϥ,cmt8tQkJqA%sqQO,WM (pxT+5僪z`T,il3K4. \7h|s6rn޶M@{B17eԩ*z&'([8bgNp&""_ ;j9|GHSnQeGD^+E)tnEm[ S lYb'c: *z{( ݒ4عe!]yC%o7]a:3MP.S՛8g@ۭҍWc CۖRJ9JK8'@d)#O;߻gT𑅛/|&R^U˫'&_rjmޟjWzLn~)Yy1̵AJ9(g~zdvaHD ;vR@N穮jI7BDD4Q""Y JSR4ZV@@ƨ"8nt\ EM@Uhv{;'}6O&"b⺫"+c2Q6͐x3]Nsܐhr+|mAf]]ڂ :*0ԠHTө*tBD@2z6-CEm{kDy_;}Q&6AdIn~uî\g""L0*(=bs8crfϪI@ {S{﫱&M\ֺ\ D*p <``> SGr{ԏU 52M [y4c}F.gJo3ʒ(ٚaw^%Y[Zm( - X``5nlŋZBm f""ʊe^gb(YHү-^V`SMM'YPh VZ` U__ŠO;}UeXbvm-ϕQYּe-:\z8CΗEM^V`)=~(Xr7Y^+@DDD,ۛ1]2*YDV) 1ub4i?2FO$^W"Am*> KS DD/S:Ce P,p\uxS<ЧZm@DD,) @x@p G=+q8z=G!YTZY>[*Rg:5htxzr6g܊E8MPNP<.oN<"'O(O,UKB AsQ@TMMOr9'F᪲D!,p2Lz S>b=Ծc:&){Yf._^ntz>7sQuMpU 9Ǫo LqXqs];xX&zNÝw@DDD+lW7tQU#ľ sLJZȟLI/BY&xI Wχ>.6K5'O3[zWJ5+ Xz A (w=iDDQQQ}x0%'IәC-}<М}H`5JÑr-*rinnxt "ʮdn渚DQD ~oت@>\n~ֶDDD%gʺOWZ4zߠ3(8<䶹,|nQ\'v KL=9ڱZbsn/\:֓,Y(+PNw?b: MuBPf:eb[s{Q<)~cwTD(Tz>4)ɒ3)SD}4X]6Dc)=@> I<*4t "*UU*j)i)SLgRџ'] S,~z68t< z޻;Yt ";N`9{|N5mFDDLbM1˓ fQ?Mq41SoZS;8-hX""""ӪUvwIF9#Pֱ w= "0ozacV9JDĵXn: e/4Db[Pn At8QLcF9H$^bͶ$5+йg CB… ܳYT0usxcPh̀:=[0*N-QK?Ӂ'kvT( d-xҔLr'Ӗ\а,ш¡?C_mJ}tVE3[KU3į¡GE's(zYcLg!3"5Qn ^Sne:HD5%sdmJlxtBTQbF{ajUb?43 tƁ%gխ% zX3>{O) 뾝Zf:d կaBsTTT+L|5'ϰTTDya-wTInnjWa(MdII9 YɐhUE~L ߫4c )O L ET.VGt46;]4pKcn: )Σ Xek2)H>rnֿ """ȵCd>ө)]Q "NjtT0įNg9иDpKIBT ,}韅CD%C ׉t,kvm-IAH<:X8"4,9Hee|P= j\HJTޔL'pNVaBsRY^7= *Α *rNs[/O.h$>,6x ֺ߲ZajzvJRSdUZs9i"N4c:G.zӹ)9*+t"pm4Ưګ$.0tb-R3e:߈ r1p2%'EN~HlxtlkVXm2U9Y{ut,ēy3$_ U/ *8UY(,9ӤztUS)L<:c5cd~&5q>V9|MTqG DDD4*| s34%N7ዀhug,gUNq\а,јN+~t]V{1\'v7g iLY LY]M״$ۮ-)p""a;q_6HpWS[:1Aі+M0e,NrxX5Šw}%"UYY D\Tb|]/ ~t<~*9@n}9&[|? ' W[FUY['_p<~Sڧ|m}0a:'=xMj:dsVC<',ϵ|9c-*DJGſ8tڮoTwp6CTLfH`L3M 2#vG1r9/LUǢ seyy]o:),9J"vc[sL-ϺR}呿zMsҺO/D*]g_뇞; |{JRnةmP~YP)YocLF;asF8gv1n&8q8yM3Nq2g"_ah|kC|tQJۻEɍclN~VYjAcPLc;Yzs~~yhDDn:7K0٪ubuK"6qLEd;zEg`W_=b@TTqlYbΑm5 ]'vW\Ga[_ ~mDH(~~1Ml}9]7"#N^Og8,8yr{kGg,*u'v%U  \C/C'G4"bJy/*pXp(δ  EYW0߬`4Q (!˧FBs#N~.}K*܇X@LS1Y0k)#|PnQuc %6<s=skuGxaA9ׇG#Ql:DDDV;O4Ǯ,)G8xi,+@tN\s~m[;L£ژI'xt%q7 K犜krd eU 3Ѿ_SJy .*RT{a'v[$S ˰]:ǚS(DĊiDC#* |LȢH(zn<奈N'<'8yr&FitP{~ IJoStvzA- 7Sib<9y Lqr|V8cC~{'2MP@ msL$"\rК!9h^TmTձCh?V$;t%ŗMGV.O͇Y++kvV5U]bNn( X*.%ަp(!R4SLg *$z 5C!7K\D+=cwTTTO7&횥]QT~  bOtcĉG5#m/< l|$w@֓;vGe*i[ -zLQY&=bwFWh: QqU׉}1lO{MUniTƄz5\Y^n:P**+dDy*؅Cd4{l72l_z3bər&3֦eS-:m msms19Swm}9ŹzLq~faG6lw~;DX&1|rj(Ӊ=jFL;k`a0(4yLKt*۵)Q1U @< n(tk0`"Dr'Jy z%isMJ;vDzmQROk];p;tTIDc,9tvN[ 0DTpNNB{⇸Nn10NN8YrJ,[tYRbҿsT P"zXĉOyh3Jϰ4׎_'3QA*hK㗊,ʫIA wȅ"R=W7TubkP<И ϛ@_ʼ8˦✓))^r<8=\Nh OsL\,Џx&* ognf5TsPf Bũ9Y{Dxt7HM,~c%4C. 횥'v%hd5įRg^x(~'lOsĎ^!rә@UO5(Ͻ'jL+:S\'({4ʩ3̔*3ߦ8]fNqGLq8h?-l_fm"""nUb,"8rq(w#x6Ɯ`kS{-sPM'T _nI]<ҤjN>Vl1'J"Zp< LEB *8t&:ǺN7鉀`+e@p]ny=&.CWć]wU$d 6x^cc*U('v%6Yp,4?(] t EU٦3M*;>ub'>;8Q 3`ɶH(vSUղJ!zÉ GL硜xP?t$"V؉]}AC9 ]ijh@E,~kpM硜8-Yφh޾_ ";y3rUدB53M!67_Si'S;vGz|_hJV#N'!Xa&7f:Uf9hX;RWںa D;zN?;z[>r;A9CiLt<DVRܔo:ߥ'Rf:U9Yd҉?)I)W0IPyts%MG2¿Nj6'>tl E[ܦ1;Gx2q)9LIY7J\N+U - R;ꅣgOLS jkI>¿S8c<ȷXi: Z(njoO\;Oq An6PN1@1ۚ<*FWz?4i8əX$?imQOqVK=ܳ)8PPj<{7G:@@VA}jz}\7'(Cz ҳOzc{u+JP/oQތrw^rmu< ;R yM=}i*}oTv) "PQ)Dp>r,|wυuUσQm DDD4n Bt u"iny= \K^͂yzhXoxTu>}1Y.PT:* 6B?^u /8v+f-P܀g=}MYxun(z,D庫"P&<ɂ3偰 uK؉M,8Hy,/WqE*zXpT| XÁqG3yeo̙-STtmÔғCo?ڳm0ܓNe` =i_t2~hutpH{K=EuOIxsE⑊:+:+${WtNx!κowB,ߏ7ru<4/mnDDDX(NsvMvtg7euɟ΍CT $9hHkJ-_6ƶ۠G?& !(oY) o:D]"l ]t8GOxgiB".J;ѓ'b&cD>UUˌ];.[“@!+ONbY&jYLUn2B@v/QyseyPX/*Lg<x%N E4e"S]'vwk P ~s Ⱥ]WTq9LD-@kǮ׶"zd엨DT+\vk|I=/LWEP7Բ28ZA :=We!=VohAoYS,rp8EeM~ ;;w:tT.+,CWtج = [Wh-:k"{-PN`_tޢs (:,0-WѯνV9j~a{rz핝)Q&m9|FD[uF}"ZCvֿ@Cڮ)7wԿb:P-u:M8 IDATgr뮊AI3aٲvuxf"Q.|`y|VCAC rK-n>) QʱuxbYh)84}gh9*Vۿ l =Qxݿ* {}qAD+$e~NEddx_1øbLA'ӯ[mŹso[;&◑9˜otVoBgbNCg6v{e9k{yX,|͎CgC\ycǧs%gsz/ ߲&ެXBgClw:c:k!^&tNBgR""wK'%]|n(ph!Q=z1Zyӏ"" 6nίbmBZ )5CB;K = erte!hh6[e9Ǖk?63Ӹlܸ6c0Zm"^]!Zn޳/|nl"dJU?Y+7zn*  = 1řF;rУEuqf52Do =HD$ooWyh [2-ʶG!j49oV{Q !PFrQMmŵ̴ƕr ~PP0Enq^QxmqV{5/tZBg{NZYlkcC|rC=ؿ/[r{ ſ,jgu1}mgV9 y(Ȭ-!9Ypk.t:4%tήe91c ; 9˄vٴ 5Q4{~XMK|u>7?z~QTo]UnC0Uyƍ5jEuF_zE 砜?ƞC7GYw~гPJ/y%Zn|Vb3uCƽ,LܢG4>xu_w߭Z#z#2Q;xlгiNˍOtkܸxEg%/0џ'pL?.<7C)r)t--\,h Fg6:6:Gmf:CYMzM;GAٍgҿfs~ ui o_.^K&6Bltkl> % M:c6\.yĬK$mCg5|I6t$Yѹ umߓmu,-ùř( w C1f.U""W7CP0>y]=Q/=|!zZUh=n  E-v)m ~?\Ll)CBoH-@? A*Ϭϊ4Qm^N 9iZܼFDDHt*GJHÆ #g L@9[&fEhYbhQ/D!*Yg\yEj0:^ז.t6݄h ,t6{,t6& ժ.6Y@qeBg&tV/t$YEƤ"tvۚQ]las8tBswW^Z]烉hT}Vi@tD~lySIR䅡g K= ;z ^QU3pizq\tf.%rjySCgY4DS sj!S"MJ%@_Ι6%%{y]"bxݗBlVޣЫ}o yt5h{x`-,Ω_~o+ry7> o4.jY??ϡ~O0R?ƌ@ոZmK}P\4Fzu!z&|TK Pn0:[]}-dhй9 ]vB  }^9E.@.-våQsمBg,&fdqfq-j,t6Cg%r3ͦ, MmvǦ)γk7juG󇈉nτc(p̲<~lE탘|}nA)Mm7 = |{ӈFO;/8?"Z.?l4&3htɯnl}q_EYW-7~ Ja"Of[x?V8/wr[ :K>s33cFgu:BgKٵήk:1۸l7&s6VمVhE4,78 Ln֖^HlYΕEЫ sMHBg_܄o!ϸ_hTzݡ'"榥+opFOֽ#dEUgwLj) =Q?Dq)#gore hhM躧j-u=_Ŗ^ E4lq)w8;,DCj7[7] ._{vZnG7DDkE.^_os}/"IzKq㹳{ZZ>ނ@gĽNxӗ|#ٵx{g-Wc CB4cTnǍ'{QT7聈F"z/rxp?}֚z"ً4.6:5C4a79BgrzaUIcWR-ynNokxQs1pN70{†y Ad3Ρi:k:fCg#e::[%t4bvǘ~R/ 3}/niAi[TUkKj-cpYJ“p#z7iN5"Yp%o$Y4r6^MD}tLq<W  4h8 fhD"gU#UIx.P@^ D ViכCo~އB_TsW0RCB4D*z&Is 3hR3Oj=E6x+y|bLCۅ^9[;qas,YY%9 abd{R]بYl~-"C6Cۋfd?F5͢5KH4t3٦6bEPyld:L .b691 ^tl&6sbኇ//]me ""ط#zqnl}PעL xzÞ =fRǽos`7BA4{v|τcщKCACjkBC4jٸY+9zv$ hk&-F4" #%{:3$6S~qTT_X]w =Xm="7n/fh mƅ΀d^mn6"{r[u|Ac")MKxqBAD̳O H$O! a[!{SQsY!hhCSOYϝ-o! "Zy ̄hQ\ztf9^{ HD+pD׉x.Fvw|O`D=0 !rzAVFZmwsduq /DD]SGD.??L?ȕ#y@<ע2w9%bE/..8CgCv6:#ߒ MOu/gȾZkcL?Jh%硳blRrz-ɶD#;vju:t9V:g: [Bg1kKll)t %tv6);}ÊWԝw~w=ǘ9ZW̜ U 73C1w =QH{Oz7[B1T-yKx7 hT+}GL J:UVw1Jply;x>W3IFJ_>QЃAAۃh~~jON?[l Gqsy6:߯@:#ߢή)EBg͆Cg7i@j$aCc .t5@$ :{ۖ ۙQոM.~CgͶ2:M#eAgǶΦ:#lC3h$& :Å2sSYb3LDD4.,oc(~d&?zqsPnfygch">CzmbJѕe =|c8-46ˍ{Axݧ:衈M[g!""ZE@\a IDATh%DZyjέsTz=Q=pl>-"YOZ3rRWACR l՜;Ͻsz<Q/=by'g@Ox*гu:}1^-7#8 QxE!nq^ ma.ή7#dfakņЀGe[s!tvϛZbfBgY%tvՆmC,6g5mCg MK Cg53k.tֶ64Cgd3ZBguol(t&SKWc/C#b$hK4$"1 =Sw|{A}7;o =cDžB~x!Y _г}^ͧAtDTGzކ4[n>F i rU ZƍڽP] z *Eb.YIx2_C)V~P#eq*г-I{ViN8;i-zUo3b,٢Is$k)^06PN/蝓_:k(]G9yϑ_6/盜{=5]6/΅0%tǶ z%~z\otNڅƆyD;U湡 mVU|Z80n>i{m=W+ϽMD? t3Jס0P7"4Tzgwݿ(ksmR0Y ͮYņv,ZmRx-BŤ4ר:%tF7ˢٽbc6:6*BkwNLvtYtκ8tbi0:I:Cgt=V"DDD4,o 1)WfʅVzC:yM9 W~ Hc[hT'JB3ȩT'O?HDDkN(_pw>ݼ_/qѰyb|rU*T2T;hL7nG'n η9.:sijr{hh(^ЗDݻw = S C1f&{t!hx@mq/~DzjC ՛8;,DDD=0qzYHpbؑVNT?P~KsjB}WmN} D]&67E3CJN/(QL S8QO(p|<[XT^z""R9"j-(ѯd89p18q.rsw/}=,{C,.@:{4Ru r^ ~,7:`mA.Bg>+clhm7E Q 1n.:+ q>{a}l 3@9ۆEٝhqksڅ:k:OVY """&{υn3S"sxB9u{w]zafJs0ro}"2M!tFJ;',DDD=t^ !Z?bp!) mx1vy9Sl9&I:gk /:{P91Ga^ `o9z ='> h =Ѱ۷[wʛC1VTMR1hx o8'DTrFz33BBDDcWㅝz!fǴ>ojSܯoA3(DEqEmzeJ<#HD=%qcPH47> 9g!"":izF4TTe[;mt9~9 {Ͷ:-mh* ^aCgdϫ yƆkF:6lVmbj5&4t,t9 ]p6DBlcl#tĬjf/nNMBgM4 %;{F J֘~. =4c{nSP;zY ƒ}{a1ւ}=LC̄6Sг 7H8E6qmj!ѐ3`>."]Q@\=r/HD=!GH5Az""ƟC9iؠ8/Wyv{[Wr.nCg䡳Ms.tzUޅ>zjOEzq]whTJWalVz"~9 fJ&[z""""ZQT, D}12wh%_ЃzDzZ:=R9[n}zm=-+ fs ֍jC;ܱ6gs:?fF!t3C] Bg,nr:qsx1 s5 %tc|:盞x^,I9;d3څZ Ř7U/):qun= hd2kЃP#灓U;h-s )c"z"~u7ڛv_( =Xݵ_=&zqa4B@Dk""(,DDDDDD`>,Ҝ =(Ѓ4'}"""53 x}|9t[ 㴹 8w{6:kKlrR/Dnt: mqV39:#{ͭ桳Q2Æ&CgYabVsM\:#,.Jvss91P50jš%t..g""Ѳg~秠z.;;J&yLƏ-Dkݞ;%wB1De. DDVn@z"""""%)άU ""qNmC0%BBDDD F4z}v8w86 W5t.ntN#eYypQBg{q[na{lHw 5Z դ)ݸ(tΎOK!t.Ë[CgU1\/tV|wn.t6ec 5X6&4(t.Fxo$'""LUT^hY"7?CAFoh4ȻBO0&TmWꕹ+!xM9J“BADDDA[!r'/;_P = Q9.6vq1tj"߉:2Ϊ镲vBtECg;ǝ6: ͰV:A!tzDPml̢m0 MqjC,t6PM~9}lv MY {op3hs{CAԉBA,!8/ zkFŞ_z"^0=E =Q3%$=[Cʜ&Mz"""`Lmmqn1;wֽn3jy]Xlm 5 :Q&i9ʶ/nq^"t9߰ {(tv6gY՘f<|s"]ݦe=^:(QHb ICg|:/,% (KTo!v0>sڿoBA4*T@23t^&uh֦;DDD42^1nzꎈOxxYV3 @8/ιg\:0X+ZBgYEٵ,:^OY05\_4Z)ncsz1ifq{a[ -6lFCzM oskZBg${i:d[PMs~/p3Ѩ޺ ^zEݽcDe{Ɗ[50"ꅙßpW9F^d =כ5\`SYVa- ""1^!;ոV>9DDDDșװlq^Alqnb/M1n\ s.tF:%t6Cg'Y ۢ1tv Mp(Κ6fǵ.T6;FycT: :LCgCg]:kIO#Inǟ'""5o_=)&QF΃w^zQA|*#O"͉cO@pгZyBADDD#ʍ3zZZ^z"""bLC}`9*[W^nrq%sURxmQ~dﺶζC܅m,Y<,6Jvnivvq3X9n[vKCgtr1t]tlw穑~_1hy1 wH'm!FվC77cԩ3Tm@z"""""=Su!DL4Mps2=f~z"o9RKгHz"""SCAEʇ?&x1r#]mc;nqF}Wyk,y)?.ntVueQD ݖg Ӈh5kKBsi,cdg5$ Ƙš4hn Q:38QNuDZ s/6vmBAZ);J h>W1hH4'jqBBDDDDDKyֳFs= jU\z"""^B@ԭ8/ey[JIŹ4QXc>L4hvJ\ skOۦ@U^]\0_U5j4(TMdǧQݥQ6:q?t{a7 yQ9Qb7>w M{Q:C0[1"<66X,t60&3`5Δ.,ήeCg56Vj r3YZx%_Ƌ[4i Уh GC@DDD4NӍ y}9G1nz """ .F/uGOCAIwVlsƁu$z"""""{`a ""șgbˍǩBADDD/ibtsw/}nqjsKntV5SFg{:]n[,hBgFɶ7W0mCgPb .h Cghq͒B4n6u5k~k(څh G*P5i j71ÅyFYݽ]cl!3BADDDDD@z"""!b{>9gFpjr{m[p?6+⼔AlqYŅHCkBeb9sBtd7P("o#t9e1"WRDj_WhSB" *!!P#@dY1v(_8-@z( $a(JI)mMwT" %k 9:ř۷P:p = a?ڿCA}` =(iNU1(m8-raG*wb"KuCw}DzlFM&tHYYP YPI-B3?6ρ. %bDGuQ>FIk1)D g)p&BcJ*[k[s = Z}Q?*c@~@Nݪ2 "'0%ELYpQ* ߈WGu'3-%IʦTM^)+)EwEQCADDD4iHufʃ`nqn5 %{iv8Yb^lllg2u^7&i, nu} 'YX9 4I1ZHmgK7LG6jVD5j4t'42iXlR46D 桳-~ 0d)FPoJqG桳-j?t6YLDDD+o*ͿCBkpBkLTo=(7Oh|Po޽cϠn|®$Os6gY6j (6=7e̜b<3;GV~Z) xRۺkzEx!* TE<"f +Jբ(^{LI2;7IJNd53y4Q}S%gU..x\˟gO762BUxrP_oTl4qa!ڵ=,9w 1z%gC]qc6ӵ|_k֕bX:6EgEgm,:KPRXk;ZtztxDEgDE(H\/:rƢs/#*7;ъr(XY/:#(:prBayi-Vm!""ԛ`V@kJ=_Akƒswڐ:Q<}?9h`TP*#M 0lmD""ZbZ_x_,Ut WIUD&޲e@^h`~X9vJɻaz>xb^ "ѓW: (27aC$#/tgc|Po8_Z码1ԠXM*lߜ KeŹ!ъ"v;+EV~9,A3mZ/:kӢ3 KԈaɺ^VcLpNr/VtV (Ex9oH9k/)e茨,N^@G \d?Pmg"""Zr9{ȎYh`< ^Gj۶'l6:GoDDDDnjL<Z砾q~z]Mk=uw}";37 ȶ2z=@CC_,FD 7)􂡍G.Jfg\-#; >Ʊhpܤ ֍kôR9";߱e߉dxȜAOuU-[iv7uՓ88 `Cu%w0gsfsF܋y;F*fX3pu$2˼)np@~ů 2=Wsm۞EgkzS=N ߇C;Ò3 48mOWzZ~p3~Eg+@GEgQxxAѹ^PU8֋%h%ðIrpU$VttPPXtƇTefIRH, MA9H Egy)\q&""J3`l<: ?x9hmf6<#%xc#ӝVy)=~Qn ^~m7G <|qQgr /p1;8:CA:+],rF688E`əh^+s\ȟm7 3Q`s[B'7sY k}ɉOo>"Ou?ABƣ.?˘˼n?@NSW m۞Ed99g WK۷~vx ͫ<%'{鶑ߛtBmO}|DD<8 8/7^8/_P(謋5g"\f (,W=g=٩#_' r3P/:+{ =Zztsu gsz q~<_g"""ZB%|t'yu_"F&!{xV5!HF]9 -޷J3뻱M0o؝*xSdnN[o2r垸@Ld o)V'?Bao?o~|:6Rݸfdʓ-*6 ]=if/^/Ӡd VhXr>ե%˛˸_8ofŹ͋U%f VesXt}HpBh\>,:f"#hP|` Y<~"uDDDDI2Oʂh" 3. ^Da6笭&;<qለΓ*tK2ymOx솋 <{\Uwi[km{cj> o L饢>zVo:Lݲz/8:|&{ZGtnwu~1?Γf*xXr^{>A]Q2kPk8o˖{y5w/K48:ԱXq^ 8/ZᦢsTܸ&:6V8GU,87TW \ _tz4P/J ^lΐs8 +:G ,Rtvo'A5`EZ=Q΃sY/cvQ,R3L7L]Ng(nW[sPG J[F'&T xو,(;^(O^e$󯶌/C=SܥAz:c7XI+':amgv笃Bta96PC=sX$_?Az:glΓZ>X9 m T-ݮfBt'~y^xhDD4<D ފj׽_q^* KZn<6,:W5t&j>#vbr!AY5:6\qb34(ZCs /Kǝ#u.tjq u~^W ccW[hDDDDbyb(v[~{~`:$:Q"u`u /Eգ+WSTpSUW(UL[?|?Q= -ֹzRp^Z+Us 1,7y\!%琪j:9_b@opCsBG>&-b>EY'+(8T+YRK{<19#"˹NKY~&{K2b5P&"/~ˬP'ʫUY'R(\yns * !Qo'R@U\yg &#"J84s*0< A6B|2D}Ʀ2 +^q^EY::GsAt+Kr9Z8/Y^b9~Y!~8,.7W_d2/(:͈v~D_XtX~7*:[~6i*:Eg MYKC)s<%X<7!gsw8w֠ ݑ|O $sV`:{[Y5Wlpu." ɯZy˓ ܧFvTEp r$:Qby TՉ7nߨX硎䫹b53|:Kڈ*K+4)KϷAvkR4rMOn 3ED4Bg6NR.|dw+Œ3EeEV۝cvZqnw/\b9yzYרDfi.:^tEGEg)[hYKx^Âs1Wc<6aYyH# IDATUF-hqG9=o<: X=?L#o`u!( ?_?^72s ښfX:ˠ ^M::*z.EDZ'n#Qd:b^E^\,gY)V'0>:K(v֔T%O*:KPUx؈> PGzu~R>=>x {muAuHT]|3ە+~ P bjnR`?U3?ٛ}SOh@-]Rqy-Eg "+QVجbsѢXY'3V5Zo|ꜿ0PtFS9xk*:;('A@ вtg"""ZuئW귭u3)J?l # OԛBשfgCe`QJY _~]oo'91֙V^BRs[l_߿lsPb%{u~T=>A^j%Eipu./ɧ ûКs_ҏ[F>ϳΒ"޺): @aodfe玍L|DD? ֙9Tܺ4:Kԧlyc,k9~lVtpuYEfѰ,0,2׋Ψ3t ŋaYegxfzsl!څ,4\Ugq/:u["""@莯K,dRu{!8:CHDDD(*+7 뮜h&9h*TT'$9珍x̯D4XnW[iՇGOF<*5Q}n:U X=td,9S&AU-#wz"YЪUمJoN߂qrKGOFf~Y>πr̙,w^ND4>z Uef|[{OO_{:o]qny9+΍zX͛t@^d&]"|eYpU|99  B@D栺K0{l2WJ3i.8,9S_sӡK8(bg?.5oOÕۅ$_a8,:ry `/5Gb|- ۅʮIjx?Q_-_ฅEvx?&_ """jp5w;fzKoTe~lv5ADDDr7Tͨ? eYPHae.P͝/O~J֡k4`ҳ⼚sˢW%(,G`]?aX*f4 Q9[(' 3yV*^Sʃ++ցK*Eo; +a1Kl9^tn삏IXt"cZ/.Ņwש6㗣kTtrP"^bJz(bEg6Ț¯vF)LU:uI=~{DYr&""?eo)Varr o@lݸdAj4bqTk .QL2!ٺsgˋTZT9@ @|,_}: ӡg~w`Ns7U{c# [ ""^i53Ki{ZBeB=-YiNabuԷLW`+-6sG/:kCn0o.Egm*:z 5*Zfx9FcQݧsXn x[^lGe^u2oA:C9u"""+==P|8x4#gZgE] SBr\qScRjU!LzujIE̮[ bޟ9R1Bu!]o 9%T*{2 3os$-lxA^.wBe .3tWZg!"Zby޼uA 8mOW>K8/ش[q^jx,/+EgmUt&}>,:E憢/VAx$p$Z,y+1kx9u0뗈RÀ:uw:u'ruD3QPLJSսYǡ=~PujyGy^YCgvnBDTq_ߩA r׬C$5:G D(Nv^q:H|@:-BÅr1@ƣ:GE+Z{uZ0buP{ՇcLBD9X =7XD,9S_e]֊Ƕ./~EXq^* [JS[+.#N4TtGAw5-:r .G:DŽEg Jbs69 ҮqnKDDD<g:LbeNqXGH2dXr&""wS&=1_}::d0l"*|%v.\ L= I,D.rviv9J |:-p˺:Dfr_9L!9?:GRJCPuj"y91XE:G=:@?:f Y9(FKGw\;(<׈Yf爈ԛ(S[T-/K](/@WYcαŎ5(,KXâ9q^uc~9aK.LicċEg0 /-G%fyrDs3xr9,9 y5<:C X ""J57W>ZGٙQ?Z=X]bVP؛/Ό</Xg!P.L^`#iuQEQ^v@\ Uum9nӶ:JpyP>Lj|u"J9ќcJZGt,9Sj⼶u+hS]xŹVW~W[ֹx,DsQ,(:YlҲFݽL.:U*xa:(AGha좕f2DE%^tv9%Mg"""ꞩ*uˊwCP(Co@DDdG-Vr -#9.ϓ*{nBzőb%~/nZ{6(zu (P(O^e# 7u$sZ{ueSLg4rI4=}_ALUXHR/:GQ> s@E/Y)TwE (QJ \x($Kg[K|k+Kz\Nʊjצ+MU׬_,D; E(qxQ9: αҲyTEg.ֆsc9\`/\SE \߸h9*:k=[q&"""ZMjlF:s5kTB^lh-stF vj5*8: %+uRd.@ڛC$*2r3gPe䇭Pggv?PBD"+UrQCXrЉR^3֦n9XÕ;Ǝ ck x ƍEpu͗E砸/KTt^5{\q&""SW޿X砵Q{3[sP%9h(TgByϭs?;0YsTUB}Y(~]C$Y~b=Sս\""Gs$ H4YR{0g/_YH;Q\m#ɆuSs$7PTP͓T}[g!tPя*&uKǸZV+\t*/(:_t"_V+*:kѹo'GEghl9%^tWr 348u3usYâY׶"9h~]!ǜc!p:Qꈼdy:G&rj. jMP|:%\>Qu;o"J!j#`h9:s9>לVwIwNΑX|nزmT_P.BQL^)Y(>Uy- Νǒ3+_EvW[>Vmst7yαB2ⳄEgѦpA):x 7U]1QuFp ZzYҲ Pt/7 K>\q&""np3Tc))$2f!Du""di¨RqS |ř|:%WCAm(s1pkFT}@ei "ZgH,䷭s]f#ޥz]#|nΑb jKAߗ N0c;Ū`ə >(+-nۯ+-/Vnܰ,~XkmXhYQt/I $Egk2"sXjn[t spEWJ~f׭ a`(besQ幃Rb>Ae<:Gy̮[PC,D"o zT09RŒ!ҢP:G)` DKQg[gH`{9R7j"iQ;s'RMͅ$ %ӓYsY(YNubəW뗹H'\n#{d}!D~zDҚBΑPamruR`@G7D&Nxk$Kԇ:xy3e+K=V˹++*^wtŹp9wi?oףו$:^t"(:**.VtFpYcEfDhqEg ҫiQxT I@ٽ7[ 3|R2,9u[.gB'm( PL^ozOƣ:|u"L ]ҦH }:Fڨ73$[g jE蒡mcCXy\pʄB89F4TYp'WSIwǡ>ԊOQLӊRǮa9ԊO+%9pȈ@%(W^jV=MpZ'^4 XY%g F%*UEAW8'@=,^Їn+PzPHp[Z~ZKDDDLݹ7c,֭#2n#%$ccQ3eOP7#y[7(ΑF }G:yuS,?Cv{Y!R55+y|tǯo%-'*ZHRc#(oy^fuOg'CMG#R~YB~^YΑ6ãG;W: `uq6wZ0i!HZQ>Ŋ3{26x(c9QLV*9Wۯ8/o!ɪW^mXZ~OPp=f~Ysp_N} x/:E9 Pt:(~{6i >Z[xYgH!gBN_՚u"< Puם9҂/ԣսs,U]]y ѴR+zyWNI.Qi9| HXt ~8v mk<Tt2ᚊή{gEAB_>'""68oAKѷn:1nlp:d!ɆkG3Yqəc~CP E`=Msg*k,D߆$) iV*]}-9O1e(WE 96DM}:DXHÅjP\rVъsc{cЊs _/:^lysPtF ,4GEXys/:EqQ9\tv~Y_lB_o""""궚ȇZe9sd`ɹ6LD'P:N!LOnAݣӬ3^YwԿ yw:m(n8{C |򹅈ZH+ C`(Tܠ;sQ?ʓޟtyY/_~\EբF7 7Eg =HCix֋qs\ ~FE8uh(:wFfiH;p8Q7,lVNGw:fyX ""7|R>b/2_}: 쬨 |g]"j􃃕k~gwt7}Wƣ!@F D=)Pu$S|uMs 9Rb*4XJC 3DԟT| iŒ31祎ŊR6Ǧr9؟Yt֯P,"J+GEg /5AӢ]|:6a[ 4µsEtXtn(mGEjVnӦS[sl9uuapɹG""n~jbu_pp*T )1: Ln[ >"u䛚(9JD/@bqߔwYHuB -D.I% >;RIs xBS4XTo<*/DݥC_V,9S_Kʊjצ+-փ\SyaJzshJyEؠ|XEy9VTFp s`q:86/:/|LDDD0?`:ɶL[’spə'3(9Dc9P<:G(^Vyu[ P|:93$a;!NDi!Q#GceŹA]qn/0׋h*:$("pxYp~i9:8((G@s Eg׉cʴ,3uזSAkW'9w%&V#"02(1*\fKS3csP|4"[s7Yů#$A1걈I%g_gvjb-|oi{n!ul#%To:O6/Cob:Z]l!"K[\q^\qGz 3ɍE簴ui좮 ,:ℷ ~[}Vpְ,3\CyAScg"""Z#ODZg<"Ĝh:CDD9tU Ͳ4pzu47E`V' "q}Hquj"C:Ϧ c8s.s~#qtT @em'Αt |P"hMQ(ޛC_L}+ˮXNJ+K,u]X4Ҵ!{EgKyhl9#^tEyEXuEh߉_"""%lxm:c#':u[oQoA<:p/T': >slDdKUvYg&"R7H}FHgS_Px|n-u﹅ }9:%Ö:S(_`%gK]qnq~]qnyۮ8/vVE6E  hZtzהCzA5]_³"̡|s /KptǪׯ[DDDD1[MDnAݲe^)s@e:i:(܇7YgHŵhJCXEzniIG"3_XdQ?X "K!i@iSiz"Ƕ_qngpű-KK A98$ P@=Ċᢳ65:Pl#jP6c Q ᠪ1s״JQ" g I牸 ?oJYլ3$XDnuJC:O#$bucP򨪊껭sQ93By""Rb(AT%>t5ZH5?ӠRyuseεATzVw9Ȋ~:ԯ\qng, WWS6]qn[[q^>E6ESp]Ptآ_tF3%P^K=K8/뗈jldKo:3 87zctyaѹ4(beB#DYUEXrԊ mӴԱM6Ux9y~Eg ef/VtF+:Eέtϗ+DDDF27NA=uWSJu9+3:% >hT4gSK]N>S׭([ cjꪢuZ@wΩL:-NTH_^q:\rC|<BUܢqݶcb[7|8QI%@qD["> cə\8m8_87 EgEg/Xt^tr\0btp/,5h^,?/rř@Dĩ8!,d waɹKƒ3ʔup_r$ɣ3$~fK=B=QZsF*/[t(V&pu"bY K4@ܘcck%^tE(s#Zt 3Ϊ᧩Q9^toP~^iX-זχvdc#^f@CKDr_Fk#\qN ##$uguP@UfZS#BE~&¯uٻ8Tt>@V/dzz/\TAAE@ދ",$3 DbGd @g:ڻS|TPs2l&ZW4kq xк}8uXm^oAgΠPi_t^o[;輶y?-]s/m?wY@:9?r0]JK]:!vDGHEg`KDINg]΅UJ|i梁[̒oFg@}n:@Wc ~f -RE- cȹGn9xt(ۏ]s\ al|86\gq#ƟIkMѵqWέ׮:/=HjWa닕WKniՠkښS_@`itnoE4>DyãSLwDGȫ$i BRat 7'Ɛs{Otk-K9٩ y`3sȇ;~{D@}Fsl;,[dX>1&׽sF@OApB0ys;wN n 5;lm-Z~A-΍vZ9Ŕqm([Akq^fC 7nnrju87xк&6-[YA5߃-m4Łd[^nyqAXim5ҋ-Znwm 3ULGE@&Ĉ~#:ŹQgG|:6åo]΁KisXt+Pt=Ősƥ-v$,M9ׁ0w[dM]@ƙ1Џ'\͟({+9}ߏU 9#c*Uiq|mQwu m8d XתҜ8=ܲV,836:n~yi/2 R{A3}&;ve.3;G晋!޹kt_0Lӓr))}<:}7r31`mИYsȃxoqn%ȸTh=#.g CȶuCT>F}}jq-^V[t!DkkWgW˃뇚Wv$Fg-G28-Xݢs &:: ɹN۾l}=:z%n뾣n!:]{s >`:i[:S @p1nbEFs -3{Xt\bFGCƇg CȰUiqRs38l_t^w^bAD%ɽrW:/YTKξu7@aMm^~':ϤL왉΁1O@!0{;FEHtqyrrʔ0bYOKr }ij'?6w>+xta}YÐ3-iqnu絏}emfk]lbN4:/=Wktey ʠgJ WH@1M7Gg@JiqOFTIrr "̝A$@}_<|}P{9^ IDAT˸!crFFuŹ&C8DAƯU͵MQn +69|:^t^nkꏿ֠sڲ@qLy弩gE@熔!>@ANt D 7@E:;:CJ|psǯD}5:ćC@*`⃧1ogdfEg&>aԍfklB8/=mqp˪F64/?jtZξ:lּRڼ4ܼ<輾͙g:K΁f8}D@g'%#rPHnƐ36Nz`C_,|t%? :C{ :'ӳSƢsȆTEgO!crFeŹ纃ִ8׷ڃū-̒/:ZF祧V֭i}@~3 &G_(׽s` M,kC3{D+I@4I-=rd=aI]o<X7E'}iEǣ3`ƹ$}bp[,Y*HR8 䔥'5 9#{2\k\-5vq8~c$5kWWbL+-&_9MVץk7ƟIv~ /ɞM4=>;::vKt@=|z~1͸cXs&q#XYp+@% gdt( C(KyoIS>>q÷CsRy MÐ3-Ϋs\jv-Ͷy5+V֋]t^mm^S~yi¹| -Vj7I΁fiW 9vsO.}7:Kb ō`]gX M@~HS\>HN}B0'&vݽ׸ ω>1L;Le6Z7672-~&5Z[np ELjL052hirC#)͜!0ӝ3M_,;th+Y8p!:pR[Bj%u G\ƵnkȚEu?5!g 熍-is õu q -΍tZ4nqnIk3߫Ͷfy<謪A]J=bʩ5rwfmQo΁pN+_tVtnNgf郣3@?73`%@\QvDGșʑ#X˥oFg]C m(?N- !ȗk]guDd;o !rFdŹٶ-8^ۼiq[Ud[tÁD-΍pkQ{kdyyե΋̾Jk ?KZVȹ-tnti|AE@{܍Ab@5gf9]r,2ćysǣ3`EP>\ R+f 9w ))$?܇A 9c46mB#ZFZwj8q\Zaɚg7 :kuhyi垮;IkW =Sn/΁2̞h#C@?%ΐ36h5)OYvM>ȝ-[N^ `]#ŐXsN7CoEgֲ- 9yc{ d A}{ [_rskuXU6:$W :N;[u-ʠ6K/:ܸѐ$MD@>Yjo19-:ɐso=ҭ!_fx*~X'i27dOߋ{5Bj m{ !~H{ O̹M÷E֚Is*mgCȸf-h}-Ν ^87}ʶ⼼AgS*vy}ݗ}FYt^_ˍuml r=1:˥O?7:ZsgK:#ǶLdC{oFsA$4]t]EȜTAtc~b0DG} -|qv7+ `@vtKt]d 9gCȘת bPZŹ뚍^5+?*_`vU :ki9?Wkglv(5H#/;|Y1вo郣@?k0R&`xt1 @!U y|'+> bFs=8\jg-kܾ2t8^\{Wڞ}eyi.)Y?l˃p<ȷ=[s !"o.Ft > 2!oEg$ڪ'ce?x9b>,b֏ ķ8n n6w-^V\Ag_tş-5:/:Agjta/9-#pE@?qrlgcnߊkK0,ɹR7lanTs߻ itrFV@! b:ŹPo[-Rshqp :k4:絃VݵT{h#3ȟ$~(ҭ9И[ Ew=/!n+etr*KG Jj|z1*Rs8Ź[nn9䋃K0/yee\t<:PS\ q87m6/-Ν ^^SZt8\YU_8a]G _.ii^~W~yt友,n- 59yv_ݏ̿!L9W\ IZN+Ɛ3ROö4v%!2*%M1=hqް~8w.\oۖ[ 6jq| ت^NŹZivWAgwSdL 9tՒNHIo6tktfDg;[WhrвDƍ` "t*: ɹ*r}T ׶rF5jPo[TL\Źƃ0q osת[ȗ{W5ŭlK @LzX'&~7:jsO!/Rr1:F.;NF'u$bdf 9@1lY9#D|mQwu miu9&UykqnBp.ǯ]7xD5?GZ33)Rב58=2{N9c&s@|cqM&&0d)F0{µ.$|')cF87&#ZVH-Źlym٫}ou_oќggjt橒actlt`I-:Gm]uE8?s-]$&C Z -}iqdg.I<nyRos nǎM@àg6 ,30Jt<1ptwk]6%!~=Ɛ3G-Uiqn븴88FkQwKs̋+O+f-nՃεlf+\stSzX7Fg(GM% 9hɹp~ L[eC "ROO1iimPUyzyf:輶lqӥ;%[fZ7ls-Dٟ{oȥ!NMBt&Btwt2 M] 9#Js 9c87m6/-Ν ^8wq8-6:ʠ׺AzyhqH*- ¤?GtJ\49Y E_?3`I 9)2S N$\6rU^J87rFfZY[sMm wVZjsm ǯ:otʓ+ξf_83 ,~L96x?UwI7G?{6;M`q#XMDrOY&.$!2Ҕsz!g ޴8WZZiqnwm>nbڵs+MAF紳kI0ΰ13>:.=nbts`  |b!qTǣ3ttOȑ|tI5ōkk̯!hqnthqZ5X3ŹٶZݠkD:񋧜W5:rvYwr_"@'fۢs`;C?3:tAŇS͏Fgș٥[C597 nӾc2s"^c -As}mqnyкq7Z4kOnέ.dx57 䶙{&sF9j}Gl=/:lҿ}@t!;8J͹~rM59Ymqm6knm8w,[,mE̝8rs#]:]run̪8-䎕 I46a.OtH^eIs )n#LHt٢GgY"i{tE.qmi rFŹζ-8^[x16.pVZW{EnqW]._yiq(3Sңs]5)%oݤD(~L{htTj`r+e6: I[s]t,:@ĵmdIDgcSYUzz:n}>~sZZ@WgiqαGOscylt Hn>:CAL(btPehЧCLtm:;:Pmȍ7䉹s 䲻Dg`r`pv6_rskuhsnS_tf2Ox՟; #-/t@7-gڣկ-_]V =S;s`u :zfLpt?y0:VnJ0)Ǥrtqz!gdOUGs'Enq^vPZlfR/ @^M\xO%ߏ]OO@Kܱ8Y3Lzܑ>Ó׺ƥEg0|~,DfdP~y9sg!q?9 @m⼂5;Ź1\qWA3EIMEzLLot"sTks%/Ty1bLƐ3>| h>j9cwD'ș-[uZGFGmL:Ð3-έkŹuZV Ź7ijl&@mI*難CIP'@EgșG={*`t\q;7:Pe 9 _ir2/O+\>`g[k77^:j帴8kx}fm-9>xtyCY)-}":C$ ??:WccɤGGs]4`u CU ׶d))n|'CȸfŹqlm?Z71gO &˧\QPt,G(ۏ]sI_Q&ũ{D ϐ4#O?@.8:@ޤ'Eg-{@7\2wm#[g!g Ziqyk;mqZuܖZ80(vgϖEn 9 Gc/Qd dX~?:7f;Mzftx|":\%:@ޘAeG&$M@Τ0:CkqN@?Dd8wqPg5 n#8Z-Q-*Qht"r+vS\IitsCUREg({%:'#?/b~áC{o`d,+3:m%|gg6-: Ig$r`fklg-~&ZQm[1L\'Dm@=rBt2. .+:OlI*E5wN [l=iC3Nt4`t49C?)mwD2: IC"Dxh0q:nq3i)߇uvemvkkR{St#>bC/P`"x<E>HٻG`j|fGChre;!IrcsgH:#oDVXx@0䌁f۶;TעFk;kqF8FAiqcj4}$>2ƼO9r$6(Oq6;wk!\؋#Ѝs}':|Hi辡Gw.@KvvEG$MFg`Ns-5"نomjZYs#mqZ5xL326v%9d;0L=9r蟢_Fg%@Vs}5cf\SD(3K$D`b>a:8YsصARnr4`M^FKnḴ8 NTdtc[^_ =!?w3c-~1Q0_3WFgȣJ~0:8xNߗt2:Gޘ|6: !?#Kw}(?-i::CӖ5nYs}lm-uz:nZG& D &{Vh/rc"H[Jnog3 #DSc/tqt5:p_EsK=fǢsvGz*':FB/ IDATO#א)u--Ź/-Ν ^8wa8ͷ4e+Ǡ{\M:-I.TC$,suYt2AI9_w,G\#\ r>5>2:kj{J_tgR}':B (6Os!gdL:"`Z:niqްfŹV[2Ȁ~SOG(JbɳݽGw!2{ӎQ$vaIQ@{nt`LzD!JrԒFg@q)K~!PLoP$ 9#S\cO_YUhq8hPo8v@Zeܷ;9 ϗ[RIN=̃άo}It m3 O`^#>7woG? yd3΁br!{Zi7MhօKC*[/VrF֜֗fC yqhsKiqn9-Βd47k$QPҒ2^BfoErp?"#$%+3ĩ/tVtȃԹCu( *PD 9#;*hŹacp#0 [wص٠jmŹaעqsL 6X[0dcs# Z>t[ȓ;@٩WJ:=:GA[jƎk&G7֙9c{~*:GQx))Q\ !KJ+i4:K^]{Go.Bt21S#3΁32ףs2>䋒sŐ*M1g!ZmRs͛v%^u VkŹ-΍&pkFz\kCZ*ݧ`Ϟr.1~[В+͌ƨ>;rQ`JJ8D*/s3:97&Gҟtvt_n09fvV==:C Q W=]w惪G-mwCskiqnz篙d;`fzz,Ƕ;wQ7L`bt既s'[:\@鑙].;p/F'h%_5fώC<{ĥ;Oȷ'K:#:C߹Ϸڏ6_rskuhsܸͶ8/>0 @LfsWn$}EczTt"8zG(ߜ@Gች[R):KNnKZt;m55:Ӥ s!:C]efύEƐ32{Ҥb`ӭ׆87}ʶ\g۾8ob8~zAmt/59>8{o w^nisfn_tv|}ΊΒkFP.FzPt䕿8:[8:i1~::C^x$k-ΫJs8gŹvܳ /3.9 "+Ʃ-sGƤQ?{zl }`>tJtKR{ @L yg}m78Lg3ܽ&Fȟ]@7I:#RQtSIrF6]Xhq}Z~sתZZklgd 64>>s??':G#~JrI'z}SWYzDg_=>Lz9"yKRJ@$I yg/23~O*yyI_ΑkىGD@L^9%#2䏬7ۚڍŹkqiq--Ǜ6B6Ru -Kq|M 15)9̯I'Og8:K#{f9 ⓇztrȾH#75뗣C ?\OrC<{,|;L`p}s7*ŹPo[-Rshqp|^Z;nvn鑭*\&٣sСxC[[u<,rNM]8#r1 /@/O5:KDgP<5:G޹۫βrOg012$IȀd|fkiqKs'Enq^G?v}im-8veEeWk w!I/1!I:N%|f0ٛ&G<&:M;ʳgWc}4:F;sC`M~KsJ jqt=}lfm1+s1xOhʟhq-m&[M=n0|mt؋%-:GQo<kcg+-\CJADf(Nw=5:D-$[$΁gzɯ{T۷_2i [>.Y ?y0:r߻`9 љKc`l/_xW^Ȗ!XH*iIE@0XnO[8+-ͶmAկw>jUh8Rs#Zn#87[y ҩkcczt.{Lt2s}':ҠEML̞#6etςUnfjɫ$=4:K+CܯP&;$:KJWul^I 9 ɱ=2=q&{\tz 9#̗{H>L-5"نomjZYs#Z<-u_iqmxm`vV7S$`뱭!IKv$Et4}]752s @+.NIߢ_vF+vG?qht!08fbd<(M9ӬYT-uz:nZG&ɐ6v Q9ǤߏQT&c#ًsPb̸.C%K^%)΁ Im+:Pr\RP8tSߌ.sH 4bf[/%MFge\#7Tyȶ٧J9q3/BرO}F[ŹZuhs;iq^Yߝ}ڶ6vT̮4RW鹋7g}-ߢs̯Fȳ;_MwUrĞF֚}IvK/ l- -:kwڛS.W{o=-:DU II% $M\4Lѽ7Fv#9 ω=+9sw7T>x?r?J"@1䌾%'VyU?Z 6XKsm85=Of-oPtfmQoQ`R:27:GmO*yv_siOSe6;4U}!i8:O - tut1ӟn/^166{RRt`Gg(>\ȖC-[҃sc}~{e/Z} -͆zKs'ס--fŹI8W}z.Ɓ(ϓEn/=th9)F(_,>,:Dy/u IW WMU2AtsP}(:C u g6m]|Z4y5D(ߜ}jtdgq}u-.PtڞqFtܥ&4:_GW^Fz\G_[[npܦ- 64kOnM7 vZ1\96PfR+獠+Ctuuӧst)-It<+%΁nGv_avޖ0ȇљK}I,Xe;p/Ff,|PEeߙyqtcZ@7?G(,׽K'.xI }BݢÐ3zdU.iۚl*ump-UhmŹfh`iA kh`po-΍6h]k!7ًϼs(=[OE(*^?7wst<\RxNWss: kseLOԶkǎML~>帼D;#pm7fICYAZ @;|L{ht֎H* 7Yc=)&ͮ>Ns-ε׶ւA-nxRnG&pkfxs>JsO|m q<:G`뱭sGK,1Rt:x"{9Ж*U>?] <ӣ3Iψ΂\o>0 '䯣3yzl9;c?-O?(ikt $\0|k "Õ3: 3 9t;ۥ7{N#pFvj8q -mCʶ88F⼼+=}Z_ @%k%ME(,߻կ:c*6BIsЃ&?y8:6LGA͞95}.mIth1,;'˻;:oztA IcYp߻ wE99~/2;odyevQt@rF%!{'>۴iU6mZhv-]Gsk3[.~ʎSjүD(* G\#^L]wΑW^n?3驶S'FgA6]VyWpК[71~gfY^z{; ,1Jo(+yCi&::D^%YБ;ӣ?Q9]|〛:2:qIlKNBH VRm RFViسe [B٣!aQBvȠ@ a2H,ْҹ?ez$rsٚIF IDATq$\?))LDDTnt\zW \h:E&.m[/7É8|bM=?7t?,zj~kF*e:%'_.m7?օg%R[d: ? ̆t""""""""&N@|e[e5L.?hBQL02HdGanŹ 9*Z8o~8w}/'_j&""zj9vY7b04E$jٷOj-4 UYXo: e75&1#Q/53h 6K[_:t*A(= DY ]Ih66^\r?d̐wW+@]Lg!"3eԒ}^p[zБDl[n NZR9/[w[0M ,۞E4=?Lp*UhntⲬDMp]l! UKբzY" }35;Ck/U/ES.Pf:ݎ.0hܑLn~[_6W^ h_P ej xtfma?tJ]{BLg!"3e\eYУIc8=ZF Ź?f[7iRW0ҧVt"Lh/*S;ϡso: L>MsOeý(tҘ !"Rd:v~J|OC- ~RQ] 3/.ݶbkN5m_sWqF=Tb[A0 !N#f9M|-26ԛ~svZk!sM:s!N0^6"ZZ毩v0`FY$ZMg)D"u |ڻ=tʨr\z]n:n [uwe: bLӪ=5 H 4HD&z#GB0+(\gAD4W}7sWTU. V"{ݞK](3:Όyݵ7-P|*}5gB /$5zNv2CoKvz|(✬A~*G}rwF0"cƼv{ 7WZꇠ^ٜF?Te1M='QZ9zui/C{YFĚ@IGBK!+\jPUs9sDnyh DDEE&*AIWLv\hڦ#*w*w]k: Vr_]10Y(@e?+&綴,\m:P!CRKۛO/z#Le wчmE>nw~9Δ̻m^zH5\< lqƀZn`CN>*0DDD9+oPl:S%+uNtSb^ tTFc"s4A!wUp}FԼsLe sV %>]ZBv0,7AD42RS}{=S<Gz|v5U ~Z5h06vosPv><=a<`kǀ""""""""plC*[npa #{[ `KyҲd$cNhu-ν.ٞ1{,{}xڊ4嵱u7Ƽr]ۼz< A8wU,j=Fg""W[>ntօ iqloMp"}>H )dWӠ]S![>nMuEힾ o|WTp[b:Q6T-7sPZDEfCoo#LG"Ksz5 ә(5-- WCF}UWMWU>OKP `wy<9Sʖ]g8TŹq{r^[Dnw,;hp;.&RiCxWnqDDgrЙQ4d5N ךzN.^OstkM Fz <vWy/z=5T׎5,UV?=VշgTh>Z7([V(3\~>hkӁȮ%>@n0t&"""""""""J_r] b:U{j 0'_T'_?o,dJJ@/fƍAD}Ϋw} /'ru6?f[7^((Tkȡs8[C7#Zy=8tQﭘ`hQ ΖhxMg7T! 0#Rd<HD>RŇXGF.7/"bcԥT1쨶~"{m79ہHj!L .9ܱP'w*/rW0\?WUPuAۨ+z]3]m|'埽Q^Y^箹"62b8W/ՇbE77tl-[KI*CpDDDC[;=q@Ǧq8K*k{gy-f'9<͜o{DJ 7|QiǍ|&| -]rNfujcJbl:}U_l3 )̶Pc XUL؂s[ 2d {F`ɚ7k׾ޜ "S]Ϻab EFd+([{5cZ YRfYr?_""'[Z}22)NkT/`VSr}6rwc;\G%dJ?u!PF"{[u՞C1˅'sUUuPqBPl'Cԧvz @8w-b:JZZSs@4 nPhg;tي)b2<A60vl:AՆRn>neP ,sl Qf]KR9L F~o^ NРBuG}\Uyk[g:\<GԊZ1].'KDDDDDDDDD䫨טBH!C}#Wn[L2_6jd?P h@8L dɎ3dŹC%?S>LeKwWz5$jqN$ z;\`m˒FŞuD~`,oj(92joýL4zß>+^*Y 7AD T+,4D?6|* !&F)FcSS]pMD&VXa%ǚn98֦Р `_]M&l4pn|5B}@zݲ(7 iwUҶC.ZUgK/vyPD4PZ/]sUCsP[7:QPues8@ ϩo5kǍzgBDDQ H9HٟM'Hý^wttBQ6)x=C٦o#sOg\""B-2(z^(b-Fg#9mE1 `lKXŷ@$B;;!Tl:ZP-jCuWl,k1 F^}6Lg!|T:Gjpi1|zX@ح"XbK; ] R**nT@>@U-#'4#""89wybW]O C=fZs]|&՜}Q{U\DeŹb'O>ĭ_|ɤ!""0]KnQKVꜰDU* -_:Dpy/Rօy,DD ZL "E7ڇz,g @tÏ5tyvh@TgƪPeS`;?JD!M߬֍. m""zý,9v¡C)D8jϥp&2ilGr \ڊZŝB y|NK- Qx~ s8[0#5G-D.6#9Dj1rYL\h6(U͑y !2t"""l␳CCvy> NShm+h-S^JD4XTu̜{@-nwuAx^DDD5]; oLp0[!*O.ՋDLp$kǚQTgƴxBDdbZ5(BL """"""""""JƯM """9;P㎳ /?sZ-ΉhPlxwNmHue9ܮ"GkqToUW񵋈]T\rw \h:E>[iX s8TZ7! ]0ht",lL]^t"| {n#9R6RBl:QpPA![;ͼւ.d\*YhqZv8 eј}Pvs .ZO:CgDD4h|5b:5ŬM(H˟,3ázj&QBu|9IZZDDB6Q~EDDDDDDDDDDdTsd_xt"""lC,aaK5E0Ԟ{;Ș1s{RD7 :o4-=GwD8tꎉOh:ʖkL(˵1"3érHm.GEn2(+#ADoZ+DDDDDDDDDDDR].6/IDDD!Y48k6D_0v#hq8-8LGF;iFg-=#N86ts@xC0árצC8Asx_m: 5K!򕺊~ """"""""""5@3hqȹ@-n֘vw8$@bqMyӀ48̓D-:)g8{=!""JIE[Mp2[U;IRUU !k5ЩΌxt"A-b.ƈ DDյ"zDDDDDDDDDDDТK6h0qȹ,QOT EiTg8Q"CaiJҳ9MkOв9rb;z-L04o({]T5á C85 ZĖ#QYo+ DD)ADDDDDDDDDD`p:5t"""!d{kǧc',%u? IDAT˙g8QZ?wZIA1#>cq"""@f ~n:iL/3`] l: *Os8ASkO"*zzs~ DD֙ADDDDDDDDDDg`,9{d'_e/Q4ʑgQ8_ǿD`c3u}Ӭ+?d\u_㈈(%"S]mHCG7|b:G!62w5é,"es8HN3hc0DD$ .4(R*Xn:`XeKv|%;r^X-"p&*Pf?4ǣ۠32lm 3lwƥo|s iʽL{`K#e6 "tڮm(*]@1b.tݚJSZI11 rKK-$A'"]38q=!GQ[G,3ϼ,oC5DMᆷ}?!#r4w uQ]u'CM!"J h}M!"rprBDDDDDDDDDDuw+T7Y2ye+K%}[uK0nte\aMJ1Ĝ-ΐ v_:4BQ5>72 ^g–3WO!6J<?0b;!,fueQvis] ` N}Li- V@AFl4z@4e"""""""""""cTUQ"|j: @]_jw"@\Pd1#ŹvYmqN:LylT`]g*&7jϬ8tI*0+ZguYXuQ,qE?Tߞk{q_]C1Qߏ'm֎Gͯ V}?oڔC=^'ޯ}]퓮w[}I>gNwlf]ML6}( 6` 1C~V6z+,~Տ%KZK@'DDo$*pCA5Ey/,NFȤWꜰ,N27K5VS'M!"Tg**j ` w?3gp8CŶVnMY#l?s 8c73 -81OLZZ=s8^RU5qG).P)=ycDPQ|MIx!?"""""""""" Z>Lg!"""JWN 9mcU;2:r@mΑ|[31göO&#"ڻΎ"=f9wFsO~v 88 Yū.2#\ϝ)7$JɆReyhl4u7c +v9fŹf93d?~S{W!?g3eh__Mp{7M /m:c\\YYN j: /ڦy Y%fه@,DDDDDDDDDDD QL """J&k$GϿ@D^t]*QR:3 9Lg!"""gPm5opq"'tޓCCcd9-Q%>Qf̰]wPs]#@ rsc}':qStJH'jq-]6h-hU ?#B\ћ`ZcR s8MCWe:B)FBDOZ>L,k@m3׶Mu>ylm6Zs<[{<x#$ɥ*r`h!h#%hHEncP'պp\>"/BΌCDDiXdufYR\#pP59/O-@=as8C_޻>HcQF>ăݥ\Vs"lm6ZY(-6+hso[XvWIQȑܑˠg:iL/52Cuf ^d: H/4TkGCD9a,=$t""ʾȼ5:YR\#^1h9/=D>#YD{g[vgMi(m[as8w8k:SbBUysPw.!^ iL 43 -֚BDDDDDDDDDD FG(,DDDD@n(26tgmV[31{-_gR -<p[S8/[ɗ8ȡrKwί8!S(^tktUm ޓ=e^W]]3k 5<(bW=Yh@(P/CDDEui{0\w^^Qg[}Lg!"""אУJpM6Cci;И,{4~rJDY7૤-q"sx]#I r~ޯ6ySte 92񛼅ܥbWI( qs8![i:eOsaLp-\i:%CDiR#g0(DDߚ" 7Aզ9ٺw5ۦv!UC ^xWu7 YsmH<4Re،6G:7>[-=ӹ GLt,lzcO%mcVZquw*lqxmA&ֆst_z<&]n˭XquuVX}, *.۽Ot/6n[n~b~<>i<%2eފ4)>o7eDLp*Q=vO9(9պH喍ZCD)"pQaDf*P`,DO,9t """""""""62wyYtRi* DD*!%۽ [9vhTuIDDA-S-)4ǽ G½/2fl 6(e bc}ذO k Z^pz+1UumU}V]nΛEk IDAT*mگtunc͵*=o}WY\&RltMP :SVʠyC筼möCrP lbZIhs֕ Z9r)=*l>ؖ Ealv#쪢r_(gK~Pte矰]+%}zhgUw] `r8P~xKޢkFLd?;_/_u3Q`=֩g?dHE?VS 7$&g)u͏RhMO)IV]"F#qmXO: J(}z=9N)qSemT]Kj`d+R6fnȑ0=3l^?G/$wncYN mru׏E8ҙc6 c tBuLw]w5E1N< LVFԕƖf= +v/,sD(^#B#fv|G]ŔŹ'ߊU;fl/$puJת ;[fXt==8o6{(_,[m.0LTjNն\z*DR׿&- j}Q'4;z`b$6|۟j_|fR4^✺AgIn=ɴIqNVn-6?~0)QkČM.)3J+[嶡CLq^cL1mJQw ,E1{\+J[b%M 0KZ=%ELE]]m8'!y()[g?rJrmERcӭtvqƒm*fB?$:"0zJ*S9-RE75?o̾VK osa)x||E@?7#IkVsҜN30wژT[$Ջ~a^4?o^)Kbo|Hze38sp֮+f7 !9ilg3Y,'8ww>o|Q{)y99ks|\1C3^n%U5D9nC~^o a҆?bKW{/;L`B=:^truw4jϖZ(`S6w7|bpJ}g\|}jcy449#ݽRnQt=@[lUm̦n`M0b PScӓIq%9i.Mcq J{cG*)@R`A5FsG<Ź}R6I7.9:sr6Ƀr ~go..u'nْ:_W_+ OK䠹EpuR+O7EEƦ>Vt1dQi^kaْTt-@zp|mJeV @Uwsn/$,GG\{Ն2vI0"o~܎rg7gsfr8gXMcc&zg> {fbM)Rp3lܹFش]sRzrg^3G͓zrrmƮ#nGmzS 1BKA8~"0E1[.|nE?g t-`\Y_]y/~+\ݰeY7ܘg}. *?Qtܿ\t=@NwJv^^#R <]L&g%)!9)=9eN J1tq<Ԧ%3`lw lT ~raސ#FRn[K$%瘤uy"Kuʹb7rm\)f8 Q^:s:V+_t}'d_'0 >hf׃0wg6{^,E]EWTWt1C1{m3in_t-@W7f_o΅^&rv/Œ"Rt=@J[YzyW/mk\ 0MY~f4N}l_FigIƎFKa[(8޼n~8' ʹ!Gum'zsTcu;&q-Nq.<ش#9a9yePR~7('?8IuL;+ ,SVI͢bg7]R+__m̝J8tc19gaaF?/鮢bmg?ۏ K]cݽRnWk7-:^/$HiMRrXRzr4ݤ^w)=4(R$YqqO)1Tcʱ7rnRPRd$%[sRMZe+Z;׭$[QGk_۶<6(ҡMWfn6]GcVd[tS=wÊ.a6jt3]P}2g'ܝLj/Vr /ϗ2_p\~ۋhhϩ6_nl tt|{Iқ?qMc v)5e3LgRzm?7[coLpόސ#ylq8klӜz<rW ˍIq>0 '(mjnRt埩_(fpw$Uc4GErud6jS]I?,&`]['Ujsu}]Gs`K?i u1wS'_fg0\s۫gl`rfYU72hr>481&)fIq$ wwK͖e H9Gƶ1))ZY]uPs5)Mqnk&VuXznRs77<>fQ > iCuLjEp^:-/|^U`wgwaOuׯfgL}2{׆-KO;LWk %7YHS r0<>fFZJckPshA$0Mq^H`~L)q28^2=sRR4^<* (yF-IN e6ͥ+3,!:~%?/%E1W=8_2 /(0 ?hvyB0_T'LV5}dW9J}v.]bR=7^)?rŕUw3JAH0A(?8_2SQ@Z5]]m3+'C}値z-M;,wzL199gJʰ SG Hq6[np^?F?ESO(8g?$ͺI ߹8>Rۖ|tz5x[tSnƆiE`>l:Yo梋p:2_H1w)x .jÞ8_/_ktׯKzz0s_6Ι_ah%=4 Ǝ7f_nɜ1(GLd?1_/_^>8]C[ \Ks`e#1v)hPgIa pX 9٥&疉-Ź 8r8w+ )mk '9φ2ݧQcFyCmMoȰ yKzBuL37^% ěIZtd۶ ^t>w7|R=BHҧ%5 H+r]^mScTQ{Eׄ_8jw>Tw%m(RM;%}*,_;p }HidkLYsIJqQA;Mq.ب~ KqTG7&_Oq6͠R?ۆ26ʵa܁8>N9K"]ǔ@}NJ.!i:؉0@W곯ޠ'jYR=Ҙֽ|mϏilzUEׄ1Z{mR}~bQLr6{aE׃vS܁/ҦNJ69q;NwOH=ܶFαms +Fu<&i)25G)Ϋ%9x)}gZG~4ǯ✤)kc6K*e~6=ry%aHTtS,ta`-,l]ǔ{q[.>"P}?ڨ=A(&@҃rv|}Z `R_7LcS$E]MJr%yЀ]bҘ}a>+SϾRᖢٿKUI(MMO%ŹǦNqH%Ź `+MCOq6)Hq^?Go)vT\Y8nw=75O:1ߓȍ]a`oL?r▋_-)`ZZE`ɱE ./]FըgR=#p{GR0U\YwBq]~bRP^{jn^Q.iu+̂g1IKEׄԔ³˻.l%8Xmr6K=j)IM1cMqΖ:GD]ض8\kБy0-oNJq:OhS׿Vr]R=9|׉P&ش@l5g~Mն1[yE1,^E`2U77?,)}KE`owVoذeђYW AyX Qi^Rmu. iHVjcL&P<%jÞX(57w]ᛯp[1,ņ$ׂQ|Z>Uo-nkqH͏yϋ>-&{a\sfr8gl4zl vzi#q)L NN\;:*c#0R3oE)ssuW^))>_D&'ט{2:tnRMdk{DL[/'jtiږM-'cۻkEׁUY >(iwuL3t ]tM\gm3>I]&AI46|lQnvE0ݫ8_{S^I]7++*ٿr/,8?L;-rdWO^YYqOڴafe $)gKژA))0A)~c4')9ylqS `\ >Ź|tν!lȑ~nk&)v3ُ7I0R}Bi)-uҖA 65(.[a_^6_3]hUYqOQ~F &o,& \d Ϙ]>/>@l>[t ~ݡJcJ}Ѓ \[I@6jOVoDTt[u[ JtSaCh8gM;lC3IS@QLq{y{{:'YӬCsJsQLq^8#i&b>8N/AjZ/$m)ifosu:0*+)K'n¢oe6{eas]G" ]h,,=R={1]Hw5s3_ l6n%WHz1Mum ʪ~0:4nZ}G;_/d&r{U/4>1?覹kQ7#Ifs-UWKRsQIq;~4( `eIM.^Cj7ݐ#aQGMX87FoSǨZ\ySW<\Βw'{ѥ&{euL3ZBu`zxӮHƢkR}縗.czϛ]^:qsC r=Pɿ"׵%|Ɨ]݁;t%tqz~FqE׆p߇^Xbz[/a}~Ӣk%]{1/xטkdzzѵ!#&}Q >\i&g\o]X<;c)HqN0RT~ؽR37fiTMg3yGy%Ź)&X9wVLG6_""-ifK3*L'nE2͎۬%}:0?єs'{ɳBdAYB Ġ=(ӗ\vJTo"¥kmKuLWi~y6Ӳk$iI .)#V>Ocԏ߼:P0pK6_\t㖱Ӊ./w.Mfw]dSo)hX-ko,yw7a5 ^X o+lv0V6Z'nپ|'\5%%Ԥ,BO*l IDATnk]8s6[tYt d;s[pL*⛜n+=>061)99){"ƶk%4ݴ>>Tߓ7~]0B~W})^!ϛ$ u/dx[ƮorvI2[}垑"ŹiRpZ7Y F"*.Mk3vkuRAu9vAuD]GINkC[Zw`ϯ3kϵ[߫VzkIKU d"I/@8 BI_ t}I%X;{Kc/ M4VtMPew) 5'l9g\zg]Fi>Y]@3MG) έ8G"9bcK31= ,qgY4#9B_S*:t7?HqNS#COqYsSה6#Rǹc֍PJOT 8pC]Ҷm=#hڋB.B D&͹s8P~I?]^:煦t3o׿K9몵 ^VIJzꉡ,|.FE5l 7- h ,xrilPtvb5woI76zT$ 6g9{>.1uy#QsC{lM96= ;6y,c﻽5'^g\7iCعZϊLشRl(ڴ7G2|L-%z0w;%]#I§CIZ $鈤ig>Y޴xpDSҎ>=#/ۥ^dcp4$mwϙ/]WYqO_%}@Nx±;CXҥZNy.ZpIߔ:Aѽi5cp<ɐ8#gȨ)ΦfOaSs3;mӏ.(f8gѥ9>tYߤ<%-lΒKMIwۭR57Qm<궕lrM+_2;mqOzYp~,Iϔ4S\#c[%e76vs '`VayKdv̱džO-547=#p=MH:E#uul+pzp`V>%IںGo0;B;Nt!Ӷ*\M[][7|~bVmKx;o-O 䧻$;U +t~@vxpC. 0f$Hl8'7( *9猍BjX?@Q. #ܚ7]dsʱms79d8cRg_AgsѶR3^-û~;|e2v>#E";WZ<l[ZdS'B7c1.{GK4䐤=ncǦ>Y} t|Mw]$@/?cIz136_axdGju&>Isӷ.]Pc>iQVV|k=E"S~LO/􀛾g. ܵv*@qk!CMqGRद4i,ilbRtuHIq iLt=3>9/R߮K"9ҍ?Ǘnq))m7pFc˗r#v۪$}v.Hm嫫my=̛vL$7,%?ɤ͒N4iKH:Q˿nZNJZԐ{=hICrg%Ȥޅ6X#[m촍'n>QK (xKIdv$dI۴I[Ǫ+_*HJ/>iឍ𡇾r :1÷|uض‡op=3|PI.?YmﶷI:V}\rJȗn[REfU*p_(HQsOwoO `(f\:>]"pʆ)9{hPOXRsoҏ)9)&HS٨} ;7j 96#c~Onȑ3Bn?ó4FM!87;^P'If0Vs7 ]r/4;ۡǹfŬZ`cp}XVR33; NXivzܜRɗ$>oaa>I$ݞeܣ%[xtxfkӗ6JҌmiDJa=ɞB^nz_2M4ޙf BMcM00.ccbsXRL?.đey-fK~04̾)ΑqqmC IqΑZIOM$i,Z`Lq1y:RS3Bi&N7~||nXR޼Ad4Jq84[uц41i @!A8Ž30Rp{NqN"=zގg'=4ZgkhFߘ;O}~Lu =XԚ%9_ՠPNWē`OqNސ#źHiCuR8gg*ŹRklD ~S8ǒ PfR ~h8v8gIn3$0tLqz.I 9zy/#%Jh&nZdgo'ź9R)9b_Ik߆25ŹV{/JgsQIq;~4(P$5T'@!ҧ w7Ҩnf"7.X'>=3qHq^Շ VBgDr))&Ih8J?,iw&o̷Cr۽N8w; Hqng F ITg'5(% %9bng)Βn`4un|v5ĤݘingSsͼn#✸8O3@M7eIqDsǶ9)rَ @1%9BBuly6+uCs|_S#rC|.)*$wۙx0RS'A=UF?S "9Z׊l9sIq${ HA8iP*4Ź Jc,i9d0|)S-q{f-s'8yù>tgֿyV)ΫKsĹ,gaiJqN;}MqN^MR:&9q$L3@V/>EsHqӠ,}`LisDs0R4^+޹)Ω✾!8~tc;\\@ 9jy̗%8猍BIMchPJLf#hIqn#!8{Buq8Lq8QsF,1c[~X)4<e̴;)Z˝<䦱=4($|~o0ȦiKqbX)q"Ʀ]'Δ߿Io v=~C&}Ox ZB}<&cPȡ ŏ^X4FIƌuǴ+@q-3}dNq 9R8~ȲIʀ}l|5#)ν7^Z/{qMk%i,}@Sc8o⎴˲!Gf6ljnO66úx=\)d85,F?Ź}?fIqΟvXPski8N],)U;_uY7UuG59b8wlgM3{_x1)ksIsĹE8K\<iWѓ>78[嗈uzL&=3؄9HqшR&NG$=zHs,_;8=8'FsIIqx= )Ή19rh?R#9_:kseÑ7]G$6VsOij6E`LHsK]qO/9[v,rRb,)Pumr~w^vȤ,k$j&2ŹkO* F )Τ8Z7Iɝ\LskR0 ]%pIZ^b%9A`Js cێ.Q5os:ՏMR" 5Źd>ؑOqE69_Z៷>JscliwTIzo쾹CJ86*)ǣ-8i88X/)Y6HZ'{srhMΒ}@ҏ$ܥA)ZRX\Tf Jg)X{)]ƒnlHqN>w8oIo,RF]l_Z7,bsl J4eI7iKlX~gϙxݭSP'&)qzX'1ŹF^RD4Z=eݶė+k 0b%^[9s8gl~iu$Ai,blt$iM Hc޴Rӥ'8]uݞoq_ScIgq,%5۟XLM.R8gwj0:VX✹r)Ή;﻽7ZM_esSq)YVs=MXg,߻>|4~4( 1ŹâRM3n#&*9KpsEkCĹ7'ic-3ϕɝ9G乣FoҏM.)(B&gIZܺw%{)c*/aMqnYN8uCmu!GuS %9aNX@7ID3H|kfzfc4vS~W|7ä07H7Fnȑbl0S;J|G+9OORhC\!`jnr&{5✩z0)Ο?7HZa%8əjCu)9kuu')H3j%zm!:0R37wJ3,In=)R#$893:9QŀR#LebRxϔIJޤ㴡Ls%`:dnr>ueL3v*SNS"o}s`\Bw+a8[{]iJqnJsTݮeP&48`279K9w_zgS#Ź Jclqۛvd8X8BsF8L\MΒܻ_vH8G;)ή~H?(8w+a~8'L'Lqx)9)ι>tc)ogE&gI:{/ƥk!ŹCRXRRخ)?Wb& ['rlǹ8~֍um2)lXs S${SLmso vٚƾ8Sw_v80Hq^Cs8Gѭ7K S㚋{L}v+ )S&涭H.9&8CRM7MLN IDATi}d( &@M&zh%Ź˺HqNȚ9I/))]i.y5ԟ7i,)s$åC~zL;`ܥ^xoz 0"Rpe/)ι7蜣)ΞuÑ}֝{mR=:rR0,IgyydyRצql+6yotљ/{2@RۛvQ^{R;I㞙:9Nu>yTRȓ0Źk59KˍnBO=ǦƟLq摥3n{Ӯcm`)YS*i7blZxQ5e nk36nsG+9պCK!Os:2ʐ 0,I\DD"e^.톑Ѡi89,\pַq`H5Nf/ސ#ylql) Rx5miNq^9Hq$}or9"o4.)3]{dcg}*iW;}vJSSh&̱ +1H8@jr>ns2 Ź/)Kng~uő ~LsRG%cjjR,cly6(h$=:t;['Lq)pS)Kϼ #u0Hq8v);ǒLss{0N,I˿i1JG%Ź9{@n6tu4(>jOyR0ne@86}h_R=hTS#FY$}Iς1jN}aI~rCswez4Jjɜ91i ϸ퍻*igWy{|Ԥ8'yR։w39ش ˺Y=řgIPhz~=g_^KגHg/Vg)IcsoSE]ې#C3q;k=9zhIƦ+_CxuRhTG;玧2Iqt'9{Wқo K,_tMqΓvca?./o: 7ŹDՔ5;N״[Ro|sDm sts\Hvo(8`R\QϻH[]ơrJJNqݠ2Ymsm7u3ni*$x?Ncllsأ"U['{sd }gf88[ {.[-c$hb"4T*TrhD$c?P:`R.- cб)H+SBRf޹i""F'N" t>4'xe>>Ys8צ:SkF`oqr-xsZQv[/;̭h}'w7L ;sKݎ]7s4'On^^L'S͍g<;=N2-ΥPع#Fg\;C38ȐsW}j Ź^9yym8+洂hqu]suyl:Źjm8w]kͮSN8$'>x[Ƿ8W(8sŹ4'<|ڵgѳjNiP31r`Anq0w-Η&8ͬŹ;bS-έkk™2~ثW3XRfZ{27\ ZOo5>neq"\o-\r`1&8nyp:<yֵ8W<`Lc٦܅s/gAsi\r`8O5<2lz,-Ct=CZsKڹZ3B,B>:}6\3/zs{TV8gls~ lqnߺ=_-l3 Tmoqs(-΃ӦmqyB8~}^=80/!gk-εh_o;X:Gs}mq=GuX{A-}{gU?wR3 S}iq^Ź{Fͽmq\nQ3sr`Źv1x_ZfԬ\-Η8#B,fZ{<8?~F-8O8s807!gGDa8] Yݽ8O=GC8F-Νk{/goq|o9u[1Osvm9ŹǾ8Qozsvmkq-λ-|B,H;<׳8}-=Z[36<﷩Y;Źz81i E&΋k5Zǵ8^kqn}97hnozx}XWUARs1yD-=AZkft޻Wh"69opNHhqn"rЪDZsݒg|w!Ϫondž?{79焜XTm^8lT}8 -kqjq^ Z]A49l3n;Wޘ#BZ'8wsksZKs k+殫oqXst-)g@3 Y}7#;8_RosMsZGY8O _Ψ:X纵34" kq$ndž?79sB,[?tx8"&8gZjy0q{ ےG]kqv=v|4r`YR]6p{Rƫ 9,=t'EmRsimqD򚺵y-Υp|ܵ@ }=N%Ϭg1mc焜X&5?}e˗!kƇk8g_]sf-]Nǧ}0x6Aor`q>5"\yIgֶ]YsޢciqNqOocЇm 3ƇiR38_>뚵=\ b*LsOs?gbg"cI!gW@8_3j@Z{lŹ<\7.E1+B,g-Eg&æA0qHR߽k'8轷ͬhrZ-3[i܈ts" |ZqVLD&N[3xs#iqn ZkqMq36JŻ DJwfjq΄L8&nǵJhqvcpIam8W>k/\N4DDmwlWS uBwo#fmq.S4q':Źi9&yS-+Ź2hu%߬]/M;禹7{Ƿ=UB?zizoH#9Z ^1Ź3h}\-cZ4$݌|'tgNqߛ[iqnvMvݻpX-Γjq.Hq<M<t`!gGJ)Κ"7[&N;+{lŹfǞf8kqIqyKrvZMFBݴ867|:<-ztz-kK9Is3;noHш6!ggԻ?XKnŹctM'v4'wLiε8WW8ׇt8׭U79g5ҝ8I#ҳq܉&E*yLzS-i-=?|vT܈/oŞD|@&p鷽ONүG=|fluf6q)"$R>SD#r^wg_AɅb5 /=I,)SwrY}kg\t\qV5AM:i4q?/‡4^KO+W5q#}+֚[Vwki|9pٻ5'EE<|q5@ʙ2Ak7Ƶs>?5=ʌzDu᷑ϯXWy3Swq>7ūӇv}8*O|~u/E"y%{;Cιs9׆6^gн!癞յs‡>+FTD|C<A(;`㬉OSVvKJ-֮٣<`Z87:ޏ\՞Sjq 0wz♙OopX!g?_jW ͼRsn)ʵPj./hqΩ ^w~s{g&8oפw r(tvޛ*[y 8jsY8)5)>Mw}rhF:Źh-΃ֹak׮e[C|M|u&ήpBkNڈ=9=G?wMlV~s<[95`zt`!g޽ox'߭o"~8"93RsZvWxzw}9-Yh&^ _΅87gN9YҬͅOE+_w} <'_M<~巳ݖ\r-vKJ-Ι2Z,Wx3rsｷ_`D\ Ȯ-ͬ]o _;8x W _&lB[<%<}nEk0̧II<ُEFD.N+V2@m9r.8gڥ3g,K-ιs+LnūӇv}CFz^qMčw8kq\وOq.^>]B0ѭMM""ueZ kqWX0손3^y[R|Wehq^ -ΝCoqZqE?/[>% w/hVJhqks-΅syHgmujwrToצaBa _xyyD<">"ݻrNZ+sԥ\:|yڥg9O"/G"mv} 3lӫ9OYsڳ͍xIhs~B{5oq|hDĻ#ůǍx{+ٮr]_rS|qs/h"ҟ/H'&iDgDԄi8_yq;"nEcz4">M|$x<>OwxNG;@?zIENDB`xgrammar-0.2.3/docs/_static/img/mlc-logo-with-text-landscape.svg000066400000000000000000000375621521764210300246170ustar00rootroot00000000000000 image/svg+xml xgrammar-0.2.3/docs/api/000077500000000000000000000000001521764210300150345ustar00rootroot00000000000000xgrammar-0.2.3/docs/api/python/000077500000000000000000000000001521764210300163555ustar00rootroot00000000000000xgrammar-0.2.3/docs/api/python/bitmask_ops.rst000066400000000000000000000004531521764210300214240ustar00rootroot00000000000000Bitmask Operations ================== .. currentmodule:: xgrammar .. autofunction:: allocate_token_bitmask .. autofunction:: apply_token_bitmask_inplace .. autofunction:: reset_token_bitmask .. autofunction:: get_bitmask_shape .. autodata:: bitmask_dtype The dtype of the bitmask: int32. xgrammar-0.2.3/docs/api/python/builtin_structural_tag.rst000066400000000000000000000014661521764210300237070ustar00rootroot00000000000000Builtin Structural Tag ================================ .. currentmodule:: xgrammar.builtin_structural_tag This page contains the API reference for the structural tag template function. For its usage, see :doc:`Tool Calling and Reasoning <../../structural_tag/tool_calling_and_reasoning>`. Global Functions ---------------- The main public entry points are: .. autosummary:: get_model_structural_tag normalize_tool_choice register_model_structural_tag .. function:: get_builtin_structural_tag :noindex: Deprecated alias for :func:`get_model_structural_tag`. All APIs -------- The remaining model-specific structural tag builders are generated from ``xgrammar.builtin_structural_tag`` automatically. .. automodule:: xgrammar.builtin_structural_tag :members: :undoc-members: :autosummary: xgrammar-0.2.3/docs/api/python/compiled_grammar.rst000066400000000000000000000002271521764210300224120ustar00rootroot00000000000000xgr.CompiledGrammar ======================== .. currentmodule:: xgrammar .. autoclass:: CompiledGrammar :no-show-inheritance: :autosummary: xgrammar-0.2.3/docs/api/python/config.rst000066400000000000000000000005131521764210300203530ustar00rootroot00000000000000Config ========================== .. currentmodule:: xgrammar Recursion Depth Management -------------------------- .. autofunction:: get_max_recursion_depth .. autofunction:: set_max_recursion_depth .. autofunction:: max_recursion_depth Serialization Version --------------------- .. autofunction:: get_serialization_version xgrammar-0.2.3/docs/api/python/exception.rst000066400000000000000000000003431521764210300211050ustar00rootroot00000000000000Exception ========================== .. currentmodule:: xgrammar.exception .. autoclass:: DeserializeFormatError .. autoclass:: DeserializeVersionError .. autoclass:: InvalidJSONError .. autoclass:: InvalidStructuralTagError xgrammar-0.2.3/docs/api/python/grammar.rst000066400000000000000000000002341521764210300205340ustar00rootroot00000000000000xgr.Grammar ================ .. currentmodule:: xgrammar .. autoclass:: Grammar :no-show-inheritance: :special-members: __str__ :autosummary: xgrammar-0.2.3/docs/api/python/grammar_compiler.rst000066400000000000000000000002651521764210300224320ustar00rootroot00000000000000xgr.GrammarCompiler ======================== .. currentmodule:: xgrammar .. autoclass:: GrammarCompiler :no-show-inheritance: :special-members: __init__ :autosummary: xgrammar-0.2.3/docs/api/python/grammar_matcher.rst000066400000000000000000000004361521764210300222430ustar00rootroot00000000000000xgr.GrammarMatcher ======================= .. currentmodule:: xgrammar .. autoclass:: GrammarMatcher :no-show-inheritance: :special-members: __init__ :autosummary: .. autoclass:: BatchGrammarMatcher :no-show-inheritance: :special-members: __init__ :autosummary: xgrammar-0.2.3/docs/api/python/index.rst000066400000000000000000000004511521764210300202160ustar00rootroot00000000000000.. _apixgrammar: XGrammar Python API =================== .. toctree:: :maxdepth: 2 grammar tokenizer_info grammar_compiler compiled_grammar grammar_matcher testing structural_tag builtin_structural_tag openai_tool_call_schema bitmask_ops config exception xgrammar-0.2.3/docs/api/python/openai_tool_call_schema.rst000066400000000000000000000011271521764210300237330ustar00rootroot00000000000000OpenAI Tool Call Schema ======================= .. currentmodule:: xgrammar.openai_tool_call_schema This page contains the API reference for tool and tool choice schema models used by builtin structural tag APIs. .. autoclass:: FunctionDefinition .. autoclass:: FunctionToolParam .. autoclass:: BuiltinToolParam .. autodata:: ToolParam .. autoclass:: NamedToolChoiceFunction .. autoclass:: NamedToolChoiceParam .. autoclass:: BuiltinToolChoiceParam .. autoclass:: AllowedToolRef .. autoclass:: AllowedToolsParam .. autoclass:: AllowedToolChoiceParam .. autodata:: ToolChoiceOptionParam xgrammar-0.2.3/docs/api/python/structural_tag.rst000066400000000000000000000050721521764210300221560ustar00rootroot00000000000000Structural Tag ========================== .. currentmodule:: xgrammar.structural_tag This page contains the API reference for the structural tag. For its usage, see :doc:`Structural Tag Usage <../../structural_tag/structural_tag_api>`. Top Level Classes ----------------- .. autoclass:: xgrammar.StructuralTag :show-inheritance: :exclude-members: model_config .. autoclass:: StructuralTagItem :show-inheritance: :exclude-members: model_config Format Union ------------ .. autodata:: Format Basic Formats ------------- .. autopydantic_model:: ConstStringFormat :show-inheritance: :exclude-members: model_config .. autopydantic_model:: JSONSchemaFormat :show-inheritance: :exclude-members: model_config .. autopydantic_model:: AnyTextFormat :show-inheritance: :exclude-members: model_config .. autopydantic_model:: TokenFormat :show-inheritance: :exclude-members: model_config .. autopydantic_model:: ExcludeTokenFormat :show-inheritance: :exclude-members: model_config .. autopydantic_model:: AnyTokensFormat :show-inheritance: :exclude-members: model_config .. autopydantic_model:: GrammarFormat :show-inheritance: :exclude-members: model_config .. autopydantic_model:: RegexFormat :show-inheritance: :exclude-members: model_config .. autopydantic_model:: QwenXMLParameterFormat :show-inheritance: :exclude-members: model_config Combinatorial Formats --------------------- .. autopydantic_model:: SequenceFormat :show-inheritance: :exclude-members: model_config .. autopydantic_model:: OrFormat :show-inheritance: :exclude-members: model_config .. autopydantic_model:: TagFormat :show-inheritance: :exclude-members: model_config .. autopydantic_model:: TriggeredTagsFormat :show-inheritance: :exclude-members: model_config .. autopydantic_model:: TokenTriggeredTagsFormat :show-inheritance: :exclude-members: model_config .. autopydantic_model:: TagsWithSeparatorFormat :show-inheritance: :exclude-members: model_config .. autopydantic_model:: OptionalFormat :show-inheritance: :exclude-members: model_config .. autopydantic_model:: PlusFormat :show-inheritance: :exclude-members: model_config .. autopydantic_model:: StarFormat :show-inheritance: :exclude-members: model_config .. autopydantic_model:: RepeatFormat :show-inheritance: :exclude-members: model_config .. autopydantic_model:: DispatchFormat :show-inheritance: :exclude-members: model_config .. autopydantic_model:: TokenDispatchFormat :show-inheritance: :exclude-members: model_config xgrammar-0.2.3/docs/api/python/testing.rst000066400000000000000000000002451521764210300205650ustar00rootroot00000000000000xgr.testing =========== .. currentmodule:: xgrammar.testing .. automodule:: xgrammar.testing :members: :private-members: :undoc-members: :autosummary: xgrammar-0.2.3/docs/api/python/tokenizer_info.rst000066400000000000000000000003601521764210300221330ustar00rootroot00000000000000xgr.TokenizerInfo ====================== .. currentmodule:: xgrammar .. autoclass:: VocabType :show-inheritance: :autosummary: .. autoclass:: TokenizerInfo :no-show-inheritance: :special-members: __init__ :autosummary: xgrammar-0.2.3/docs/build_docs.sh000077500000000000000000000001561521764210300167330ustar00rootroot00000000000000#!/bin/bash # build the docs to _build/html set -euxo pipefail make clean make html python3 wrap_run_llm.py xgrammar-0.2.3/docs/conf.py000066400000000000000000000102601521764210300155610ustar00rootroot00000000000000# -*- coding: utf-8 -*- import os import sys from datetime import datetime import tlcpack_sphinx_addon import tomli # -- General configuration ------------------------------------------------ os.environ["XGRAMMAR_BUILD_DOCS"] = "1" sys.path.insert(0, os.path.abspath("../python")) sys.path.insert(0, os.path.abspath("../")) # Load version from pyproject.toml with open("../pyproject.toml", "rb") as f: pyproject_data = tomli.load(f) __version__ = pyproject_data["project"]["version"] project = "XGrammar" author = "XGrammar Contributors" copyright = f"2024-{datetime.now().year}, {author}" version = __version__ release = __version__ # -- Extensions and extension configurations -------------------------------- extensions = [ "myst_parser", "nbsphinx", "autodocsumm", "sphinx.ext.autodoc", "sphinx.ext.autosectionlabel", "sphinx.ext.autosummary", "sphinx.ext.intersphinx", "sphinx.ext.mathjax", "sphinx.ext.napoleon", "sphinx.ext.viewcode", "sphinx_copybutton", "sphinx_reredirects", "sphinx_tabs.tabs", "sphinx_toolbox.collapse", "sphinxcontrib.autodoc_pydantic", "sphinxcontrib.httpdomain", "sphinxcontrib.mermaid", ] nbsphinx_allow_errors = True nbsphinx_execute = "never" autosectionlabel_prefix_document = True nbsphinx_allow_directives = True myst_enable_extensions = [ "dollarmath", "amsmath", "deflist", "colon_fence", "html_image", "linkify", "substitution", ] myst_heading_anchors = 3 myst_ref_domains = ["std", "py"] myst_all_links_external = False intersphinx_mapping = { "python": ("https://docs.python.org/3.12", None), "typing_extensions": ("https://typing-extensions.readthedocs.io/en/latest", None), "pillow": ("https://pillow.readthedocs.io/en/stable", None), "numpy": ("https://numpy.org/doc/stable", None), "torch": ("https://pytorch.org/docs/stable", None), } autodoc_mock_imports = ["torch", "safetensors", "transformers", "tvm_ffi"] autodoc_default_options = { "members": True, "undoc-members": True, "show-inheritance": True, "inherited-members": False, "member-order": "bysource", } autodoc_pydantic_model_show_field_summary = False autodoc_pydantic_model_show_json = True autodoc_pydantic_settings_show_json = False # -- Other Options -------------------------------------------------------- templates_path = [] redirects = {} source_suffix = {".rst": "restructuredtext", ".md": "markdown"} language = "en" exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "README.md"] # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" suppress_warnings = ["misc.highlighting_failure"] # A list of ignored prefixes for module index sorting. # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False # -- Options for HTML output ---------------------------------------------- # The theme is set by the make target import sphinx_rtd_theme html_theme = "sphinx_rtd_theme" html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] html_static_path = ["_static"] # Add custom CSS files to fix text selection issues html_css_files = ["css/fix_text_selection.css"] footer_copyright = "© 2024 XGrammar" footer_note = " " # html_logo = "_static/img/logo.png" # html_theme_options = {"logo_only": True} header_links = [ ("Home", "https://xgrammar.mlc.ai/"), ("Docs", "https://xgrammar.mlc.ai/docs/"), ("Github", "https://github.com/mlc-ai/xgrammar"), ("Blog", "https://blog.mlc.ai/"), ] html_context = { "footer_copyright": footer_copyright, "footer_note": footer_note, "header_links": header_links, "display_github": True, "github_user": "mlc-ai", "github_repo": "xgrammar", "github_version": "main/docs/", "theme_vcs_pageview_mode": "edit", # Set the logo in left sidebar "logo": "img/logo.png", "theme_logo_only": True, # "header_logo": "_static/img/logo.png", # "header_logo_link": "", # "version_selecter": "", } # add additional overrides templates_path += [tlcpack_sphinx_addon.get_templates_path()] html_static_path += [tlcpack_sphinx_addon.get_static_path()] # Some scripts to override a certain field in the documentation xgrammar-0.2.3/docs/developer_guide/000077500000000000000000000000001521764210300174255ustar00rootroot00000000000000xgrammar-0.2.3/docs/developer_guide/building_docs.md000066400000000000000000000027411521764210300225600ustar00rootroot00000000000000# Building Docs XGrammar uses Sphinx to build the documentation, and the documentation is hosted on GitHub Pages. The document can be written in Markdown (`.md`, preferred) or reStructuredText (`.rst`). ## Building Docs Locally Install the dependencies: ```bash # For non-Debian-based systems or Conda environments, use your package manager to install ruby. sudo apt update sudo apt install ruby-full python -m pip install -r docs/requirements.txt gem install jekyll jekyll-remote-theme ``` Build the docs locally: ```bash bash scripts/local_deploy_site.sh ``` This will build the website and the docs, and host them locally at `http://localhost:8888`. ## Deploying Docs on GitHub Pages The documentation is built and deployed automatically when you merge your changes into the `main` branch. The workflow is defined in [`.github/workflows/documentation.yaml`](https://github.com/mlc-ai/xgrammar/tree/v0.2.1/.github/workflows/documentation.yaml). The docs will be build locally, uploaded to [`xgrammar/gh-pages`](https://github.com/mlc-ai/xgrammar/tree/gh-pages) and then deployed to GitHub Pages. ## Best Practices for Writing Docs When adding new features to XGrammar, please update the documentation accordingly. Each time you make changes to the docs, you need to build the docs locally to see the changes and make sure the changes are correct. When referencing a code in the repository, make sure you are referring to a specific release version of the code, instead of the main branch. xgrammar-0.2.3/docs/developer_guide/code_coverage.md000066400000000000000000000021141521764210300225320ustar00rootroot00000000000000# Code Coverage The script [`run_coverage.sh`](https://github.com/mlc-ai/xgrammar/blob/main/scripts/run_coverage.sh) offers a way to test the code coverage of the XGrammar library. To run the coverage test, please follow these steps: 1. In `config.cmake`, set the variable `XGRAMMAR_ENABLE_COVERAGE` to `ON`. 2. Compile the XGrammar library with the configured settings. 3. Run the script `run_coverage.sh` in the root directory of the XGrammar library. Note that the script invokes `lcov` with `--gcov-tool /usr/bin/gcov-13`, so it expects the library to be compiled with GCC 13 and `gcov-13` to be installed. If you use a different compiler version, update the `--gcov-tool` path in the script accordingly. After running the script, you will find the coverage report in the `coverage_report` directory. Please note that code coverage tools are merely aids to help identify which parts of the code have not been tested. However, pursuing 100% code coverage is not advisable. It can actually have [negative consequences](https://neatstack.substack.com/p/stop-using-code-coverage-as-a-quality). xgrammar-0.2.3/docs/index.rst000066400000000000000000000023551521764210300161310ustar00rootroot00000000000000👋 Welcome to XGrammar ====================== XGrammar is open-source solution for flexible, portable, and fast structured generations. The mission of this project is to bring flexible zero-overhead structure generation everywhere. .. toctree:: :maxdepth: 1 :caption: Get Started start/installation start/quick_start .. toctree:: :maxdepth: 1 :caption: Structural Tag structural_tag/structural_tag_api structural_tag/tool_calling_and_reasoning structural_tag/advanced_usage .. toctree:: :maxdepth: 1 :caption: Tutorials tutorials/constrained_decoding tutorials/workflow_of_xgrammar tutorials/advanced_topics tutorials/engine_integration tutorials/json_generation tutorials/ebnf_guided_generation .. toctree:: :maxdepth: 1 :caption: XGrammar Features xgrammar_features/runtime_safeguards xgrammar_features/serialization xgrammar_features/javascript_api .. toctree:: :maxdepth: 1 :caption: Developer Guide developer_guide/building_docs developer_guide/code_coverage .. toctree:: :maxdepth: 1 :caption: API Reference api/python/index .. TODOs: .. xgrammar_bnf .. json_schema .. supported_models .. debugging .. rewrite json/ebnf generation, engine_integration xgrammar-0.2.3/docs/requirements.txt000066400000000000000000000010061521764210300175440ustar00rootroot00000000000000autodoc-pydantic autodocsumm ipykernel ipywidgets jupyter_client linkify-it-py markdown>=3.4.0 matplotlib myst-parser nbconvert nbsphinx nbstripout pandoc pillow pydantic sentencepiece setuptools==81.0.0 sphinx==5.2.3 sphinx-autobuild sphinx-book-theme sphinx-copybutton sphinx-reredirects==0.1.2 sphinx-rtd-theme sphinx-tabs == 3.4.1 sphinx-toolbox == 3.4.0 sphinxcontrib-mermaid sphinxcontrib-napoleon==0.7 sphinxcontrib_httpdomain==1.8.1 tiktoken tlcpack-sphinx-addon==0.2.2 tomli torch transformers urllib3>=2.5.0 xgrammar-0.2.3/docs/start/000077500000000000000000000000001521764210300154205ustar00rootroot00000000000000xgrammar-0.2.3/docs/start/installation.md000066400000000000000000000051371521764210300204510ustar00rootroot00000000000000# Installation XGrammar Python Package can be installed directly from a prebuilt package or built from source. ## Method 1: Prebuilt Package XGrammar supports various platforms: * Operating Systems: Linux, macOS, and Windows * Hardware: CPU, NVIDIA GPUs, AMD GPUs, Apple Silicon, TPU, etc. * Python: 3.9 and later. We provide Python wheels for XGrammar via pip. ```bash python -m pip install xgrammar ``` For use with MPS on Apple Silicon, install with: ```bash python -m pip install "xgrammar[metal]" ``` We also provide conda packages for XGrammar: ```bash conda install -c conda-forge xgrammar ``` Use the following command to verify installation: ```bash python -c "import xgrammar; print(xgrammar)" # Prints: ``` ## Method 2: Build XGrammar Python Package from Source This option is useful when you want to make modification or obtain a specific version of XGrammar. ```bash git clone --recursive https://github.com/mlc-ai/xgrammar.git && cd xgrammar pre-commit install # Copy cmake config. You can update the config if needed. cp cmake/config.cmake . # Install scikit-build-core and apache-tvm-ffi as build dependencies. python3 -m pip install scikit-build-core apache-tvm-ffi python3 -m pip install --no-build-isolation -e . ``` XGrammar is a library written in C++ and Python. The editable install will automatically rebuild the package when XGrammar is imported in Python. ### Optional: Run Python Tests ```bash # Install the test dependencies python3 -m pip install ".[test]" # If you have a HuggingFace token, you can run all tests including the ones that have gated models. huggingface-cli login --token YOUR_HF_TOKEN python3 -m pytest # If you do not have a HuggingFace token, you can run a subset of tests that do not require gated models. python3 -m pytest -m "not hf_token_required" ``` ## Method 3: Build XGrammar C++ Library Only XGrammar can also be built as a C++ library. This is useful for using XGrammar in C++ or Rust projects. XGrammar uses CMake and Ninja to build the C++ library. To build only the C++ library, you can set -DXGRAMMAR_BUILD_PYTHON_BINDINGS=OFF when running CMake, or modify cmake/config.cmake manually. ```bash git clone --recursive https://github.com/mlc-ai/xgrammar.git && cd xgrammar # Copy cmake config. You can update the config if needed. cp cmake/config.cmake . mkdir build && cd build cmake -G Ninja .. ninja ``` ### Optional: Run C++ Tests ```bash # Run all tests bash scripts/run_ctest.sh # Run a subset of tests whose name contains "test_name" bash scripts/run_ctest.sh test_name ``` xgrammar-0.2.3/docs/start/quick_start.md000066400000000000000000000055331521764210300203010ustar00rootroot00000000000000# Quick Start This guide introduces how to use XGrammar with HuggingFace `transformers` in Python to generate structured outputs. It focuses on JSON generation -- the most important use case of structured generation. You should have already [installed XGrammar](installation). ## Preparation Instantiate a model, a tokenizer, and inputs to the LLM. ```python import xgrammar as xgr import torch from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig device = "cuda" # Or "cpu" if you don't have a GPU model_name = "meta-llama/Llama-3.2-1B-Instruct" model = AutoModelForCausalLM.from_pretrained( model_name, torch_dtype=torch.float32, device_map=device ) tokenizer = AutoTokenizer.from_pretrained(model_name) config = AutoConfig.from_pretrained(model_name) messages = [ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Introduce yourself in JSON briefly."}, ] texts = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) model_inputs = tokenizer(texts, return_tensors="pt").to(model.device) ``` ## Compile Grammar Construct a `GrammarCompiler` and compile the grammar. The grammar can be a built-in JSON grammar, a JSON schema string, or an EBNF string. EBNF provides more flexibility for customization. See [GBNF documentation](https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md) for specification. ```python tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=config.vocab_size) grammar_compiler = xgr.GrammarCompiler(tokenizer_info) compiled_grammar = grammar_compiler.compile_builtin_json_grammar() # Other ways: provide a json schema string # compiled_grammar = grammar_compiler.compile_json_schema(json_schema_string) # Or provide an EBNF string # compiled_grammar = grammar_compiler.compile_grammar(ebnf_string) ``` ## Generate with grammar Use logits_processor to generate with grammar. ```python xgr_logits_processor = xgr.contrib.hf.LogitsProcessor(compiled_grammar) generated_ids = model.generate( **model_inputs, max_new_tokens=512, logits_processor=[xgr_logits_processor] ) generated_ids = generated_ids[0][len(model_inputs.input_ids[0]) :] print(tokenizer.decode(generated_ids, skip_special_tokens=True)) ``` ## Notice: Generation Quality When applying constrained decoding, it is recommended to **clearly describe the expected output structure in the prompt**. This is because constrained decoding only affects the sampling stage, but we want the LLM’s underlying probability distribution to already align with the structure as much as possible. ## What to Do Next - Check out [JSON Generation Guide](../tutorials/json_generation.md) and other How-To guides for the detailed usage guide of XGrammar. - Report any problem or ask any question: open new issues in our [GitHub repo](https://github.com/mlc-ai/xgrammar/issues). xgrammar-0.2.3/docs/structural_tag/000077500000000000000000000000001521764210300173265ustar00rootroot00000000000000xgrammar-0.2.3/docs/structural_tag/advanced_usage.md000066400000000000000000000106651521764210300226110ustar00rootroot00000000000000# Advanced Topics of Structural Tag ## Automatic End Detection Certain "unlimited" formats — `any_text`, `any_tokens`, `triggered_tags`, and `token_triggered_tags` — can consume an unbounded amount of output. To know when to stop, the structural tag compiler automatically detects the **end condition** of the enclosing `tag` and adds it to the format's internal exclude set. The detection works by walking up to the nearest enclosing `tag`: - If the tag's `end` is a **string** (or list of strings), the end strings are added to the exclude set of string-level unlimited formats (`any_text`, `triggered_tags`). - If the tag's `end` is a **`token` format**, the end token ID is added to the exclude set of token-level formats (`any_tokens`, `token_triggered_tags`, as well as `exclude_token`). Currently only **string–string** and **token–token** pairs are detected. Cross-level detection (e.g. a token end for a string-level format) is not supported. If you need additional exclusions beyond what automatic detection provides, specify them explicitly via the format's own exclude field (`excludes` for `any_text`, `exclude_tokens` for `exclude_token`/`any_tokens`/`token_triggered_tags`). --- ## Deprecated API: `Grammar.from_structural_tag(tags, triggers)` **The deprecated API is still available for backward compatibility. However, it is recommended to use the new API instead.** Create a grammar from structural tags. The structural tag handles the dispatching of different grammars based on the tags and triggers: it initially allows any output, until a trigger is encountered, then dispatch to the corresponding tag; when the end tag is encountered, the grammar will allow any following output, until the next trigger is encountered. The tags parameter is used to specify the output pattern. It is especially useful for LLM function calling, where the pattern is: `{"arg1": ..., "arg2": ...}`. This pattern consists of three parts: a begin tag (``), a parameter list according to some schema (`{"arg1": ..., "arg2": ...}`), and an end tag (``). This pattern can be described in a StructuralTagItem with a begin tag, a schema, and an end tag. The structural tag is able to handle multiple such patterns by passing them into multiple tags. The triggers parameter is used to trigger the dispatching of different grammars. The trigger should be a prefix of a provided begin tag. When the trigger is encountered, the corresponding tag should be used to constrain the following output. There can be multiple tags matching the same trigger. Then if the trigger is encountered, the following output should match one of the tags. For example, in function calling, the triggers can be `["`). The correspondence of tags and triggers is automatically determined: all tags with the same trigger will be grouped together. User should make sure any trigger is not a prefix of another trigger: then the correspondence of tags and triggers will be ambiguous. To use this grammar in grammar-guided generation, the GrammarMatcher constructed from structural tag will generate a mask for each token. When the trigger is not encountered, the mask will likely be all-1 and not have to be used (fill_next_token_bitmask returns False, meaning no token is masked). When a trigger is encountered, the mask should be enforced (fill_next_token_bitmask will return True, meaning some token is masked) to the output logits. The benefit of this method is the token boundary between tags and triggers is automatically handled. The user does not need to worry about the token boundary. ### Parameters (deprecated) - **tags** (`List[StructuralTagItem]`): The structural tags. - **triggers** (`List[str]`): The triggers. ### Returns (deprecated) - **grammar** (`Grammar`): The constructed grammar. ### Example (deprecated) ```python from pydantic import BaseModel from typing import List from xgrammar import Grammar, StructuralTagItem class Schema1(BaseModel): arg1: str arg2: int class Schema2(BaseModel): arg3: float arg4: List[str] tags = [ StructuralTagItem(begin="", schema=Schema1, end=""), StructuralTagItem(begin="", schema=Schema2, end=""), ] triggers = ["...` - responses that mix free-form text with structured fragments - token-level delimiters that are not represented as normal text Structural tags are also compatible with the OpenAI-style `response_format` request shape. ## Request Shape Pass a structural tag as the `response_format`: ```json { "model": "...", "messages": [ ... ], "response_format": { "type": "structural_tag", "format": { "type": "...", ... } } } ``` The `format` field is required. It contains one format object, and that object can recursively nest other format objects. Each format object represents a "chunk" of text. ## How Structural Tags Work Think of a structural tag as a tree of chunks: - **Primitive formats** match one chunk directly, such as an exact string or a JSON payload. - **Composition formats** combine other formats, such as "A then B" or "A or B". - **Tagging and dispatch formats** wrap content in delimiters or switch into constrained generation only after a trigger is seen. - **Token-level formats** do the same kinds of matching, but at the tokenizer level instead of the string level. ### Format Categories | Category | Formats | What they do | | --- | --- | --- | | Primitive | `const_string`, `json_schema`, `grammar`, `regex`, `any_text` | Match one text fragment | | Composition | `sequence`, `or`, `optional`, `plus`, `star`, `repeat` | Build larger structures from smaller ones | | Tagging / dispatch | `tag`, `triggered_tags`, `tags_with_separator`, `dispatch` | Wrap content or switch between free text and structured regions | | Token-level | `token`, `exclude_token`, `any_tokens`, `token_triggered_tags`, `token_dispatch` | Constrain output at token boundaries | ## Quick Start ### Minimal `tag` Use `tag` when the output must look like `begin + content + end`. ```json { "type": "tag", "begin": "", "content": { "type": "any_text" }, "end": "" } ``` This matches `...`. ### Minimal `triggered_tags` Use `triggered_tags` when the model should be free to emit normal text, but switch into a structured region after seeing a trigger. ```json { "type": "triggered_tags", "triggers": ["", "content": { "type": "json_schema", "json_schema": { "type": "object", "properties": { "city": {"type": "string"} }, "required": ["city"] } }, "end": "" } ] } ``` This accepts output like: ```text I will call a tool now. {"city": "San Francisco"} ``` ### Unlimited Formats and End Detection Some formats can consume an unbounded amount of output: - `any_text` - `any_tokens` - `triggered_tags` - `token_triggered_tags` When one of these formats appears inside a `tag`, the compiler automatically uses the enclosing `end` marker as part of the stop condition when the levels match: - string end -> string-level unlimited format - token end -> token-level unlimited format This is why a format like `tag("", any_text, "")` can stop cleanly at ``. For more details, see [Advanced Topics of the Structural Tag](advanced_usage). ## Format Reference ### Primitive Formats #### `const_string` Matches one exact string. | Field | Type | Default | | --- | --- | --- | | `value` | `string` | (required) | - **Use it when**: the output must contain a fixed literal ```json { "type": "const_string", "value": "Let's think step by step." } ``` #### `json_schema` Matches content that conforms to a JSON Schema. | Field | Type | Default | | --- | --- | --- | | `json_schema` | `object` | (required) | | `style` | `"json"` \| `"qwen_xml"` \| `"minimax_xml"` \| `"deepseek_xml"` \| `"glm_xml"` | `"json"` | | `any_order` | `bool` | `false` | - **Use it when**: the structured part is naturally expressed as schema-constrained data `style` values: - `"json"`: standard JSON - `"qwen_xml"`: Qwen-style XML parameters, such as `value` - `"minimax_xml"`: MiniMax-style XML parameters, such as `value` - `"deepseek_xml"`: DeepSeek-v3.2 XML parameter format - `"glm_xml"`: GLM-style XML parameter format, such as `namevalue` `any_order` relaxes object property ordering (see [below](#property-ordering-with-any_order)). It works with every `style`. ```json { "type": "json_schema", "json_schema": { "type": "object", "properties": { "name": {"type": "string"}, "age": {"type": "integer"} }, "required": ["name", "age"] } } ``` Use a non-JSON style only when the surrounding model format expects it: ```json { "type": "json_schema", "json_schema": { "type": "object", "properties": { "location": {"type": "string"} }, "required": ["location"] }, "style": "qwen_xml" } ``` ##### Property ordering with `any_order` By default (`"any_order": false`) properties must appear in their declared order and are fully validated (no duplicate keys, all required keys present). With `"any_order": true`, any property (**required**, **optional**, or **additional** / **pattern**) may appear in any position. Each entry must still be a valid property — a permitted key with a value matching that key's schema (so `additionalProperties: false` still rejects undeclared keys) — but the grammar no longer tracks *which* keys appear: required keys need not all be present, and duplicates are allowed. The only remaining bound is on the **total number of entries**, which must be between `max(minProperties, number of required properties)` and `maxProperties`. For example, with `required: ["a", "b"]` and `additionalProperties: false`, any object holding at least two valid `a`/`b` entries is accepted, even `{"a": .., "a": ..}` (`a` duplicated and `b` absent). It applies to every object, including nested ones. ```json { "type": "json_schema", "json_schema": { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"] }, "any_order": true } ``` The same flag is the `any_order` argument of [`Grammar.from_json_schema`](xgrammar.Grammar.from_json_schema) and [`GrammarCompiler.compile_json_schema`](xgrammar.GrammarCompiler.compile_json_schema). #### `grammar` Matches text with an EBNF grammar. | Field | Type | Default | | --- | --- | --- | | `grammar` | `string` | (required) | - **Use it when**: JSON Schema is too restrictive or the structure is better described directly as a grammar ```json { "type": "grammar", "grammar": "root ::= (\"yes\" | \"no\")" } ``` If the grammar is too broad, it can make later constraints ineffective. Avoid patterns that can consume almost anything when you still need precise structure afterward. #### `regex` Matches text with a regular expression. | Field | Type | Default | | --- | --- | --- | | `pattern` | `string` | (required) | - **Use it when**: a small local text fragment is easiest to describe with regex ```json { "type": "regex", "pattern": "[A-Z]{3}-[0-9]{4}" } ``` As with `grammar`, avoid overly broad patterns when later structure still needs to be enforced. #### `any_text` Matches arbitrary text. | Field | Type | Default | | --- | --- | --- | | `excludes` | `string[]` | `[]` | - **Use it when**: the content should remain free-form until some enclosing boundary is reached ```json { "type": "any_text", "excludes": ["", "content": { "type": "json_schema", "json_schema": { "type": "object" } }, "end": ["", ""] } ``` #### `triggered_tags` Allows arbitrary text until a trigger is encountered, then dispatches to one of several tags. After the tag ends, arbitrary text is allowed again until the next trigger. | Field | Type | Default | | --- | --- | --- | | `triggers` | `string[]` | (required) | | `tags` | `tag[]` | (required) | | `at_least_one` | `bool` | `false` | | `stop_after_first` | `bool` | `false` | | `excludes` | `string[]` | `[]` | - **Use it when**: tool calls or other structured regions can appear inside otherwise free-form text Important rules: - tags inside `triggered_tags` must use string `begin` fields - each tag should match exactly one trigger - each trigger should be an unambiguous prefix of the tag(s) it dispatches to ```json { "type": "triggered_tags", "triggers": ["", "content": { "type": "json_schema", "json_schema": { "type": "object", "properties": { "city": {"type": "string"} }, "required": ["city"] } }, "end": "" }, { "type": "tag", "begin": "", "content": { "type": "json_schema", "json_schema": { "type": "object", "properties": { "timezone": {"type": "string"} }, "required": ["timezone"] } }, "end": "" } ], "at_least_one": false, "stop_after_first": false } ``` Semantics of the two control flags: - `at_least_one: true` requires at least one dispatched tag and therefore disallows leading free text - `stop_after_first: true` finishes the `triggered_tags` structure after the first dispatched tag #### `tags_with_separator` Matches zero, one, or more tags separated by a fixed separator, with no extra text outside the tag sequence. | Field | Type | Default | | --- | --- | --- | | `tags` | `tag[]` | (required) | | `separator` | `string` | (required) | | `at_least_one` | `bool` | `false` | | `stop_after_first` | `bool` | `false` | - **Use it when**: the output is a pure list of tagged fragments ```json { "type": "tags_with_separator", "tags": [ { "type": "tag", "begin": "", "content": { "type": "json_schema", "json_schema": {"type": "object"} }, "end": "" } ], "separator": ",", "at_least_one": false, "stop_after_first": false } ``` #### `dispatch` Allows free-form text, but when a specified pattern string appears, the following output must conform to the corresponding format. Uses Aho-Corasick matching internally for efficient multi-pattern detection. | Field | Type | Default | | --- | --- | --- | | `rules` | `[string, format][]` | (required) | | `loop` | `bool` | `true` | | `excludes` | `string[]` | `[]` | - **Use it when**: you need pattern-triggered structured regions inside free-form text, with string-level pattern matching ```json { "type": "dispatch", "rules": [ ["", {"type": "json_schema", "json_schema": ...}], ["", {"type": "json_schema", "json_schema": ...}] ], "loop": true, "excludes": [""] } ``` To end matching at a specific string, put it in `excludes` and follow the `dispatch` with a `const_string` in a `sequence`, or use the `end` of an enclosing `tag`: ```json { "type": "tag", "begin": "", "content": { "type": "dispatch", "rules": [ ["", {"type": "json_schema", "json_schema": ...}], ["", {"type": "json_schema", "json_schema": ...}] ], "loop": true, "excludes": [""] }, "end": "" } ``` Here `dispatch` allows free text and tool calls inside `...`, but stops when `` appears because it is in `excludes`. The enclosing `tag` then consumes that end delimiter. ### Token-level Formats #### `token` Matches one token by token ID or token string. | Field | Type | Default | | --- | --- | --- | | `token` | `int` \| `string` | (required) | - **Use it when**: the delimiter is a tokenizer-level symbol rather than normal text ```json {"type": "token", "token": 42} {"type": "token", "token": "<|tool_call|>"} ``` When `token` is a string, it is resolved with `tokenizer_info`. #### `exclude_token` Matches any single token except those in `exclude_tokens`. | Field | Type | Default | | --- | --- | --- | | `exclude_tokens` | `(int \| string)[]` | `[]` | - **Use it when**: one token is needed, but some token values must be blocked ```json {"type": "exclude_token", "exclude_tokens": [42, ""]} ``` When used inside a token-level `tag`, the enclosing end token is automatically excluded. #### `any_tokens` Matches zero or more tokens, excluding those in `exclude_tokens`. | Field | Type | Default | | --- | --- | --- | | `exclude_tokens` | `(int \| string)[]` | `[]` | - **Use it when**: content should remain unconstrained until a token-level boundary is reached ```json {"type": "any_tokens", "exclude_tokens": ["<|eos|>"]} ``` Semantically, `any_tokens` is equivalent to `star(exclude_token(...))`. #### `token_triggered_tags` Token-level version of `triggered_tags`. | Field | Type | Default | | --- | --- | --- | | `trigger_tokens` | `(int \| string)[]` | (required) | | `tags` | `tag[]` | (required) | | `exclude_tokens` | `(int \| string)[]` | `[]` | | `at_least_one` | `bool` | `false` | | `stop_after_first` | `bool` | `false` | - **Use it when**: dispatch must happen on special tokens rather than text prefixes Important rules: - tags inside `token_triggered_tags` must use `token` format objects as their `begin` - elements of `trigger_tokens` and `exclude_tokens` can be token IDs or token strings ```json { "type": "token_triggered_tags", "trigger_tokens": ["<|tool_call_start|>"], "tags": [ { "type": "tag", "begin": {"type": "token", "token": "<|tool_call_start|>"}, "content": { "type": "json_schema", "json_schema": { "type": "object" } }, "end": {"type": "token", "token": "<|tool_call_end|>"} } ], "exclude_tokens": ["<|eos|>"], "at_least_one": false, "stop_after_first": false } ``` #### `token_dispatch` Token-level version of `dispatch`. Patterns are token IDs or token strings instead of text strings. | Field | Type | Default | | --- | --- | --- | | `rules` | `[int \| string, format][]` | (required) | | `loop` | `bool` | `true` | | `exclude_tokens` | `(int \| string)[]` | `[]` | - **Use it when**: dispatch must happen on special tokens rather than text patterns ```json { "type": "token_dispatch", "rules": [ [100, {"type": "const_string", "value": "x"}], ["<|tool|>", {"type": "json_schema", "json_schema": ...}] ], "loop": true, "exclude_tokens": [""] } ``` ## Common Recipes ### Force Exactly One Tool Call To require exactly one tool call from a tool set, use `triggered_tags` with both control flags: ```json { "type": "triggered_tags", "triggers": ["", "content": {"type": "json_schema", "json_schema": ...}, "end": "" }, { "type": "tag", "begin": "", "content": {"type": "json_schema", "json_schema": ...}, "end": "" } ], "at_least_one": true, "stop_after_first": true } ``` If the function is fixed in advance, a single `tag` is simpler. ### Mix Reasoning, Free Text, and Tool Calls Use `sequence` to force an initial reasoning region, then allow later tool calls inside normal text: ```json { "type": "sequence", "elements": [ { "type": "tag", "begin": "", "content": {"type": "any_text"}, "end": "" }, { "type": "triggered_tags", "triggers": ["", "content": {"type": "json_schema", "json_schema": ...}, "end": "" }, { "type": "tag", "begin": "", "content": {"type": "json_schema", "json_schema": ...}, "end": "" } ] } ] } ``` ### Token-level Delimiters Some models use special tokens instead of literal strings to start or end a structured region. Combine `tag`, `any_tokens`, and `token_triggered_tags` for that case: ```json { "type": "sequence", "elements": [ { "type": "tag", "begin": {"type": "token", "token": "<|think_start|>"}, "content": {"type": "any_tokens"}, "end": {"type": "token", "token": "<|think_end|>"} }, { "type": "token_triggered_tags", "trigger_tokens": ["<|tool_call_start|>"], "tags": [ { "type": "tag", "begin": {"type": "token", "token": "<|tool_call_start|>"}, "content": {"type": "json_schema", "json_schema": ...}, "end": {"type": "token", "token": "<|tool_call_end|>"} } ], "exclude_tokens": ["<|eos|>"], "at_least_one": true } ] } ``` ## Built-in Model Styles If you only need a standard tool-calling layout for a supported model family, prefer the built-in helper instead of hand-writing every wrapper. Common examples include OpenAI Harmony response format, Llama, Qwen, Kimi, DeepSeek, and others. See [Tool Calling and Reasoning](tool_calling_and_reasoning) for `get_model_structural_tag` and the list of supported models. ## Mapping to OpenAI Tool Calling Options Structural tags are flexible enough to implement the strict-format parts of the OpenAI tool-calling API. The exact wrapper still depends on the target model's syntax, but the control knobs map cleanly: - `tool_choice = "auto"`: use `triggered_tags` with `at_least_one: false` - `tool_choice = "required"`: use `TagsWithSeparatorFormat` or `OrFormat`. - `tool_choice = {"type": "function", "function": {"name": ...}}`: use a fixed `tag` format to describe the tool-calling format. - `parallel_tool_calls = false`: set `stop_after_first: true` - `parallel_tool_calls = true`: keep `stop_after_first: false`, or use `tags_with_separator` if the model expects a pure separated list of calls See [Tool Calling and Reasoning](tool_calling_and_reasoning) for the mapping from `get_model_structural_tag` to OpenAI Tool Calling Options. ## Next Steps - For API reference, see [Structural Tag API Reference](../api/python/structural_tag). - For tool calling and reasoning, see [Tool Calling and Reasoning](tool_calling_and_reasoning). - For automatic end detection details, and deprecated APIs, see [Advanced Topics of the Structural Tag](advanced_usage). xgrammar-0.2.3/docs/structural_tag/tool_calling_and_reasoning.md000066400000000000000000000206511521764210300252110ustar00rootroot00000000000000# Tool Calling and Reasoning ## Introduction An LLM response can mix up to three parts: a **reasoning** section (e.g. `...`), **plain text** output, and one or more **tool calls**. Different models use different tags and orderings for these parts. XGrammar provides **Builtin Structural Tag** to generate a `StructuralTag` that describes the full output structure for a given model, so all three parts can be constrained during decoding. The API accepts `tools` and `tool_choice` in the [OpenAI Chat Completions](https://platform.openai.com/docs/api-reference/chat) convention. Serving engines already using that format can adopt XGrammar with minimal changes. For builtin / hosted tools (e.g. `web_search_preview`), the API extends the convention with XGrammar-specific fields; see [OpenAI Tool Call Schema](../api/python/openai_tool_call_schema) for type definitions. The `tool_choice` parameter controls how tool calls and text are mixed: - **Auto** (`"auto"`): the model may output plain text, tool calls, or both. - **Required** (`"required"`): at least one tool call is required; plain-text-only output is not allowed. - **Forced** (named choice): exactly one specified tool must be called. - **None** (`"none"`): tool calls are disabled; only text and reasoning are allowed. The `reasoning` parameter controls whether the model-specific reasoning section is enabled. ## Basic API: `get_model_structural_tag` `get_model_structural_tag` generates a `StructuralTag` for the given model type with the specified tools and options. The returned `StructuralTag` can be used with `Grammar.from_structural_tag` or `GrammarCompiler.compile_structural_tag` to obtain the corresponding grammar. Use it when you need to constrain the model to output in a fixed pattern such as "tool name + parameter JSON", e.g. for Llama, Qwen, Kimi, DeepSeek, OpenAI Harmony, etc. ### Parameters - **model** (`str`): The structural-tag style. Valid values are `"llama"`, `"qwen_3"`, `"qwen_3_5"`, `"qwen_3_coder"`, `"kimi"`, `"deepseek_r1"`, `"deepseek_v3_1"`, `"harmony"`, `"deepseek_v3_2"`, `"minimax"`, `"glm_4_7"`, `"deepseek_v4"`. - **tools** (`List[ToolParam | dict]`, optional): Function and builtin tools available to the model. The list can contain two kinds of tools: - **Function tools** use the OpenAI Chat Completions shape: ```json {"type": "function", "function": {"name": "...", "parameters": {...}}} ``` The `"parameters"` field accepts a JSON Schema dict, `True` (any JSON), or can be omitted (unconstrained). When `"strict"` is `False`, the parameters constraint is skipped. - **Builtin tools** use a compact shape with XGrammar-specific fields: ```json {"type": "web_search_preview", "name": "browser.search", "parameters": {...}} ``` - `type`: the provider-level builtin tool type. - `name`: the model-output tool name (defaults to `type` if omitted). - `parameters`: the JSON schema for constrained decoding of the builtin tool arguments. Default `None` (treated as empty list). - **tool_choice** (`ToolChoiceOptionParam | dict | None`, optional): Controls whether the model may or must call tools. Default `"auto"`. - `"auto"`: the model chooses between text output and tool calls. - `None`: treated the same as `"auto"`. - `"none"`: disables all tools. - `"required"`: requires at least one tool call. - `{"type": "function", "function": {"name": ...}}`: forces one function tool. - `{"type": }`: forces one builtin tool (matched by `type`). - `{"type": "allowed_tools", "allowed_tools": {"mode": ..., "tools": [...]}}`: limits available tools before applying its `mode`. The `tools` list may contain both function refs and builtin refs (matched by `type`). - **reasoning** (`bool`, optional): Whether to enable reasoning mode (``/`` tags or model-specific equivalents). Default `True`. - **any_order** (`bool`, optional): When `True`, applies `any_order=True` to every `JSONSchemaFormat` in the generated structural tag, so each tool's arguments may be emitted in any property order (see [`JSONSchemaFormat`](structural_tag_api) for the exact semantics). Default `False`, which keeps the declared property order with full validation. Passing an unsupported `model` or an invalid `tool_choice` will raise `ValueError`. ### Returns `StructuralTag`: The structural tag for the given model's function-calling format. ## Examples ### Function tools ```python from xgrammar import Grammar, get_model_structural_tag tools = [ { "type": "function", "function": { "name": "get_weather", "parameters": { "type": "object", "properties": {"city": {"type": "string"}}, }, }, }, { "type": "function", "function": { "name": "get_time", "parameters": {"type": "object", "properties": {}}, }, }, ] structural_tag = get_model_structural_tag("llama", tools=tools) grammar = Grammar.from_structural_tag(structural_tag) ``` ### Builtin tools For models that support builtin tools (e.g. Harmony / gpt-oss), include builtin tools in the same `tools` list: ```python structural_tag = get_model_structural_tag( "harmony", tools=[ { "type": "function", "function": { "name": "user_tool", "parameters": {"type": "object", "properties": {"q": {"type": "string"}}}, }, }, { "type": "web_search_preview", "name": "browser.search", "parameters": { "type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"], }, }, ], ) grammar = Grammar.from_structural_tag(structural_tag) ``` ### Reasoning mode For formats that support reasoning (like Qwen3, DeepSeek-R1, Kimi-K2), pass `reasoning` to enable/disable: ```python structural_tag = get_model_structural_tag("qwen_3", tools=tools, reasoning=True) grammar = Grammar.from_structural_tag(structural_tag) ``` If `reasoning` is not passed, reasoning mode is enabled by default. ### Tool choice Force a specific function tool: ```python structural_tag = get_model_structural_tag( "llama", tools=tools, tool_choice={"type": "function", "function": {"name": "get_weather"}}, ) ``` Force a builtin tool by type: ```python structural_tag = get_model_structural_tag( "harmony", tools=[...], tool_choice={"type": "web_search_preview"}, ) ``` Allow only a subset of tools: ```python structural_tag = get_model_structural_tag( "harmony", tools=[...], tool_choice={ "type": "allowed_tools", "allowed_tools": { "mode": "auto", "tools": [ {"type": "function", "function": {"name": "get_weather"}}, {"type": "web_search_preview"}, ], }, }, ) ``` ### Special tokens in text By default, special tokens like `` / `` are not allowed in free text. Pass `exclude_special_tokens=False` to allow them: ```python structural_tag = get_model_structural_tag( "qwen_3", tools=tools, exclude_special_tokens=False, ) ``` Defaults to `True`. No effect for models without special tokens (e.g. `harmony`). --- ## Supported models The `model` argument of `get_model_structural_tag` accepts the style names below: | `model` (style) | Supported models | |-----------------|-------------------| | `"llama"` | Meta-Llama-3, Llama-3.1, Llama-3.2 | | `"qwen_3"` | Qwen3, Qwen3-Next | | `"qwen_3_5"` | Qwen3.5, Qwen3.6 | | `"qwen_3_coder"` | Qwen3-Coder, Qwen3-Coder-Next | | `"kimi"` | Kimi-K2, Kimi-K2.5 | | `"deepseek_r1"` | DeepSeek-R1, DeepSeek-R1-0528 | | `"deepseek_v3_1"` | DeepSeek-V3.1, DeepSeek-V3.2-Exp | | `"harmony"` | gpt-oss | | `"deepseek_v3_2"` | DeepSeek-V3.2 | | `"minimax"` | MiniMax-M2.5 | | `"glm_4_7"` | GLM-5, GLM-4.7 | | `"deepseek_v4"` | DeepSeek-V4 | ## Extending with custom models Use `register_model_structural_tag` to add support for a new model format. See the [Builtin Structural Tag API Reference](../api/python/builtin_structural_tag) for details. ## Next Steps * For function and tool choice schema definitions, see [OpenAI Tool Call Schema API Reference](../api/python/openai_tool_call_schema). * For builtin structural tag API reference, see [Builtin Structural Tag API Reference](../api/python/builtin_structural_tag). * For advanced usage, see [Advanced Topics of the Structural Tag](advanced_usage). xgrammar-0.2.3/docs/tutorials/000077500000000000000000000000001521764210300163115ustar00rootroot00000000000000xgrammar-0.2.3/docs/tutorials/advanced_topics.md000066400000000000000000000113261521764210300217640ustar00rootroot00000000000000# Advanced Topics This section covers advanced topics about XGrammar. ## Multi-threaded Grammar Compilation and Cache To accelerate computation, [`xgr.GrammarCompiler`](xgrammar.GrammarCompiler) is multithreaded. It uses multiple threads to process a single grammar and can also compile multiple grammars in parallel. `xgr.GrammarCompiler.compile_*` functions releases the GIL, so you can use asyncio to compile multiple grammars in parallel. The `max_threads` parameter controls the maximum number of threads used. We recommend setting it to half the number of your CPU’s virtual cores for optimal performance. ```python grammar_compiler = xgr.GrammarCompiler(tokenizer_info, max_threads=8) # Use asyncio to compile multiple grammars in parallel async def compile_grammars(): # Submit two grammars in sequence future1 = asyncio.to_thread(grammar_compiler.compile_grammar, grammar1) future2 = asyncio.to_thread(grammar_compiler.compile_grammar, grammar2) # Wait for both futures to complete compiled_grammar1 = await future1 compiled_grammar2 = await future2 return compiled_grammar1, compiled_grammar2 compiled_grammar1, compiled_grammar2 = asyncio.run(compile_grammars()) ``` [`xgr.GrammarCompiler`](xgrammar.GrammarCompiler) also includes a cache. If the same grammar is compiled again, the cached result is returned directly. Set `cache_enabled` to `True` to enable the cache, and `cache_limit_bytes` to control the maximum memory usage for the cache. The cache uses LRU (Least Recently Used) eviction policy. The EBNF string, JSON Schema string, regex pattern are used as the cache key for [`compile_grammar`](xgrammar.GrammarCompiler.compile_grammar), [`compile_json_schema`](xgrammar.GrammarCompiler.compile_json_schema), [`compile_regex`](xgrammar.GrammarCompiler.compile_regex), respectively. By caching the input string directly, we further reduce the time spent constructing the grammar. ```python grammar_compiler = xgr.GrammarCompiler(tokenizer_info, cache_enabled=True, cache_limit_bytes=128 * 1024 * 1024) compiled_grammar1 = grammar_compiler.compile_grammar(grammar) # return immediately compiled_grammar2 = grammar_compiler.compile_grammar(grammar) grammar_compiler.clear_cache() ``` ## Handle Padding to the LLM Output Logits Sometimes the shape of the LLM output logits can be larger than the size of the LLM tokenizer’s vocabulary. This is because the LLM pads the output tensor. For example, the tokenizer of DeepSeek-V3 only defines 128,815 tokens, but its output probability distribution has a dimension of 129,280. Note that XGrammar always treat **the size of the model’s output logits** as the vocabulary size, because the bitmask operates on the LLM output logits. This is used in [`xgr.TokenizerInfo`](xgrammar.TokenizerInfo) and [`xgr.allocate_token_bitmask`](xgrammar.allocate_token_bitmask): ```python tokenizer_info = xgr.TokenizerInfo(tokenizer, vocab_size=129280) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) ``` For most models, the logits' vocabulary size can be found in the model config. ```python config = AutoConfig.from_pretrained(model_path) vocab_size = config.vocab_size ``` ## Generate Token Masks in a Batch XGrammar provides a new class [`xgr.BatchGrammarMatcher`](xgrammar.BatchGrammarMatcher) for users to generate token masks in a batch. ```python batch_grammar_matcher = xgr.BatchGrammarMatcher(max_threads=8) ``` `BatchGrammarMatcher` needs a parameter `max_threads` to initialize. It represents the maximum threads in `batch_fill_next_token_bitmask`. If not set, it will use std::thread::hardware_concurrency() / 2 as the default value. `BatchGrammarMatcher` has three methods: `batch_fill_next_token_bitmask`, `batch_accept_token`, and `batch_accept_string` to handle the mask generation tasks. Here is an example to use `batch_fill_next_token_bitmask`: ```python matchers = [grammar_matcher_1, grammar_matcher_2, grammar_matcher_3, ...] batch_size = len(matchers) token_bitmask = xgr.allocate_token_bitmask(batch_size, tokenizer_info.vocab_size) batch_grammar_matcher = xgr.BatchGrammarMatcher(max_threads=8) batch_grammar_matcher.batch_fill_next_token_bitmask(matchers, token_bitmask) ``` Each matcher will store its token mask in the corresponding tensor. For `batch_accept_token` and `batch_accept_string`, each matcher will try to accept the corresponding token_id/str. ```python inputs = [token_id_1, token_id_2, token_id_3, ...] results = xgr.BatchGrammarMatcher.batch_accept_token(matchers, inputs) # List[Bool] ``` ```python inputs = [str_1, str_2, str_3, ...] results = xgr.BatchGrammarMatcher.batch_accept_string(matchers, inputs) # List[Bool] ``` Each boolean value in `results` represents whether the input is accepted by the corresponding matcher. xgrammar-0.2.3/docs/tutorials/constrained_decoding.md000066400000000000000000000100471521764210300230020ustar00rootroot00000000000000# Constrained Decoding Constrained decoding is a technique used by XGrammar to generate structured outputs. In each step of LLM inference, XGrammar will provide a token mask to the LLM. The mask allows the LLM to generate tokens that follow the grammar, and prohibits those not. The mask is a binary mask of the same length as the vocabulary size. In the sampling stage of LLM inference, the mask is used so that the sampled tokens must be valid in the mask. ![Constrained Decoding](https://raw.githubusercontent.com/mlc-ai/XGrammar-web-assets/refs/heads/main/tutorials/constrained_decoding.png) Let's take a closer look. The binary mask applies to the logits of the LLM. It sets the logits of the tokens that are not allowed to $-\infty$, so that their probability will be $0$ after softmax. Then the sampler will sample from the vaild tokens with probability $>0$. ![Constrained Decoding Logits](https://raw.githubusercontent.com/mlc-ai/XGrammar-web-assets/refs/heads/main/tutorials/constrained_decoding_logits.png) By ensuring that each token generated by the LLM conforms to the given structure step by step, we can guarantee that the entire generation adheres to the specified structure. Constrained decoding provides these benefits to LLM applications: 1. **Increase generation quality.** Sometimes an LLM can generate output that is nearly correct but makes mistakes in the details. For example, it might output a JSON integer field like 1.23 as a string (“1.23”), or even get the type completely wrong. Constrained decoding can ensure the output always follow the given structure. 1. **Easy to parse and process.** Constrained decoding ensures that the output of the LLM is clean, without extraneous text or syntax errors. This makes its output directly parsable and seamlessly integrable with downstream applications. 1. **Ensure safety and avoid unexpected outputs.** We can control the content generated by the LLM to prevent unexpected erroneous outputs. There have been [many recent reports](https://protectai.com/blog/mcp-security-101) about safety issues in agent applications, and the ability to ensure the correctness of generated content is especially valuable for today’s agent use cases. Constrained decoding has minimal negative impact on LLM generation. This is because if the LLM is already capable of generating correct responses, applying constraints doesn’t do anything. But if the LLM fails to produce a correct answer, the constraints can at least ensure the structure is correct. **Note:** when applying constrained decoding, it is recommended to also describe the expected output structure in the prompt. This is because constrained decoding only affects the sampling stage, but we want the LLM’s underlying probability distribution to already align with the structure as much as possible. ## XGrammar's Implementation In XGrammar, we store the token mask in a compressed bitset format using `int32`. Use [`xgr.allocate_token_bitmask`](xgrammar.allocate_token_bitmask) to allocate a token mask. XGrammar uses [`xgr.Grammar`](xgrammar.Grammar) to describe the output structure (or, the grammar). It has a compilation step of the grammar to speed up the generation of the mask. The [`xgr.GrammarCompiler`](xgrammar.GrammarCompiler) class compiles a [`xgr.Grammar`](xgrammar.Grammar) object into a [`xgr.CompiledGrammar`](xgrammar.CompiledGrammar) object. The [`xgr.GrammarMatcher`](xgrammar.GrammarMatcher) class, constructed from a [`xgr.CompiledGrammar`](xgrammar.CompiledGrammar) object, is used to generate the mask. It is a stateful class that can be used to generate the mask for a single step of the generation. For each step, it will accept the last token generated by the LLM with [`xgr.GrammarMatcher.accept_token`](xgrammar.GrammarMatcher.accept_token), and then generate the mask with [`xgr.GrammarMatcher.fill_next_token_bitmask`](xgrammar.GrammarMatcher.fill_next_token_bitmask). Then the mask can be used to guide the sampling of the next token. ## Next Steps See [Workflow of XGrammar](workflow_of_xgrammar.md) to learn more about the constrained decoding process. xgrammar-0.2.3/docs/tutorials/ebnf_guided_generation.md000066400000000000000000000145401521764210300233050ustar00rootroot00000000000000# EBNF-Guided Generation XGrammar enables efficient structured generation. Besides JSON, you can use an EBNF grammar to guide the generation, providing more flexibility for customization. We first go over how to use XGrammar in an LLM engine to achieve this in [EBNF-Guided Generation in LLM Engines](#ebnf-guided-generation-in-llm-engines), we then provide an end-to-end JSON generation using XGrammar with HF `transformers` in [Try out via HF Transformers](#try-out-via-hf-transformers). ## Install XGrammar [XGrammar](../start/installation) is available via pip. It is always recommended to install it in an isolated conda virtual environment. ## EBNF-Guided Generation in LLM Engines In this section, we see how to use XGrammar in an LLM engine to ensure that the output follows an EBNF grammar. All code snippets below are actual runnable code as we simulate the LLM generation. First, import necessary libraries for the tutorial. ```python import xgrammar as xgr import torch import numpy as np from transformers import AutoTokenizer, AutoConfig ``` Then, we extract tokenizer info from the LLM we are using with `xgr.TokenizerInfo`. With the `tokenizer_info`, instantiate `xgr.GrammarCompiler` that will compiler a grammar of your choice. ```python # Get tokenizer info model_id = "meta-llama/Llama-3.2-1B-Instruct" tokenizer = AutoTokenizer.from_pretrained(model_id) config = AutoConfig.from_pretrained(model_id) # This can be larger than tokenizer.vocab_size due to paddings full_vocab_size = config.vocab_size tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=full_vocab_size) compiler = xgr.GrammarCompiler(tokenizer_info, max_threads=8) ``` Then specify an EBNF grammar string. We currently use the GBNF format (GGML BNF), with the specification [here](https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md). ```python ebnf_grammar_str = """root ::= (expr "=" term)+ expr ::= term ([-+*/] term)* term ::= num | "(" expr ")" num ::= [0-9]+""" compiled_grammar = compiler.compile_grammar(ebnf_grammar_str) ``` With the compiled grammar, we can instantiate a `xgr.GrammarMatcher`, the main construct we interact with that maintains the state of the structured generation. We also allocate a bitmask that will be used to mask logits. ```python # Instantiate grammar matcher and allocate the bitmask matcher = xgr.GrammarMatcher(compiled_grammar) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) ``` Now we simulate a single-request auto-regressive generation. See [Integration with LLM Engine](engine_integration.md) for batched inference. ```python # Here we simulate a valid sampled response sim_sampled_response = '(5+3)*2=16<|end_of_text|>' sim_sampled_token_ids = tokenizer.encode(sim_sampled_response, add_special_tokens=False) # Each loop iteration is a simulated auto-regressive step for i, sim_token_id in enumerate(sim_sampled_token_ids): # LLM inference to get logits, here we use randn to simulate. # logits is a tensor of shape (full_vocab_size,) on GPU # logits = LLM.inference() logits = torch.randn(full_vocab_size).cuda() # Apply bitmask to logits to mask invalid tokens matcher.fill_next_token_bitmask(token_bitmask) xgr.apply_token_bitmask_inplace(logits, token_bitmask.to(logits.device)) # Sample next token probs = torch.softmax(logits, dim=-1).cpu().numpy() next_token_id = np.random.choice(list(range(full_vocab_size)), p=probs) # Accept token from matcher to update its state, so that the next bitmask # generated will enforce the next token to be generated. Assert to make # sure the token is indeed valid. Here we accept the simulated response # assert matcher.accept_token(next_token_id) assert matcher.accept_token(sim_token_id) # Since we accepted a stop token `<|end_of_text|>`, we have terminated assert matcher.is_terminated() # Reset to be ready for the next auto-regressive generation matcher.reset() ``` ## Try out via HF Transformers XGrammar can be easily integrated with HF transformers using a `LogitsProcessor`. Note that this integration mainly aims for accessibility and may contain extra overhead. First, instantiate a model, a tokenizer, and inputs. ```python import xgrammar as xgr import torch from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig device = "cuda" # Or "cpu", etc. model_name = "meta-llama/Llama-3.2-1B-Instruct" model = AutoModelForCausalLM.from_pretrained( model_name, torch_dtype=torch.float32, device_map=device ) tokenizer = AutoTokenizer.from_pretrained(model_name) config = AutoConfig.from_pretrained(model_name) messages = [ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Introduce yourself in JSON briefly."}, ] texts = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) model_inputs = tokenizer(texts, return_tensors="pt").to(model.device) ``` Then construct a `GrammarCompiler` and compile the grammar. ```python tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=config.vocab_size) grammar_compiler = xgr.GrammarCompiler(tokenizer_info) # Grammar string that represents a JSON schema json_grammar_ebnf_str = r""" root ::= basic_array | basic_object basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) ".0"? basic_number ::= ("0" | "-"? [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= (([\"] basic_string_1 [\"])) basic_string_1 ::= "" | [^"\\\x00-\x1F] basic_string_1 | "\\" escape basic_string_1 escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= "[" ("" | ws basic_any (ws "," ws basic_any)*) ws "]" basic_object ::= "{" ("" | ws basic_string ws ":" ws basic_any ( ws "," ws basic_string ws ":" ws basic_any)*) ws "}" ws ::= [ \n\t]* """ compiled_grammar = grammar_compiler.compile_grammar(json_grammar_ebnf_str) ``` Finally, use `LogitsProcessor` to generate with grammar. ```python xgr_logits_processor = xgr.contrib.hf.LogitsProcessor(compiled_grammar) generated_ids = model.generate( **model_inputs, max_new_tokens=512, logits_processor=[xgr_logits_processor] ) generated_ids = generated_ids[0][len(model_inputs.input_ids[0]) :] print(tokenizer.decode(generated_ids, skip_special_tokens=True)) ``` xgrammar-0.2.3/docs/tutorials/engine_integration.md000066400000000000000000000225471521764210300225150ustar00rootroot00000000000000# Integration with LLM Engine XGrammar enables efficient structured generation. In this tutorial, we go over the key components of XGrammar and how to integrate XGrammar into an LLM engine. We first lay out the concepts in [High-Level Flow](#high-level-flow). We then demonstrate how XGrammar enables [Structured Generation for Batched Inference](#structured-generation-for-batched-inference). The code snippets below are actual runnable code as we simulate the LLM generation. ## Install XGrammar [XGrammar](../start/installation) is available via pip. It is always recommended to install it in an isolated conda virtual environment. ## High-Level Flow In this section, we go over the key components of XGrammar when integrating it into an LLM engine for structured generation. First, import necessary libraries for the tutorial. ```python import xgrammar as xgr import torch import numpy as np from transformers import AutoTokenizer, AutoConfig ``` ### xgr.TokenizerInfo `xgr.TokenizerInfo` is a per-model construct that encapsulates tokenizer information, including all its vocabulary. There are several ways of instantiating it, and the most convenient way is using an `AutoTokenizer`. Note that for some models, `AutoConfig.vocab_size` can be larger than `AutoTokenizer.vocab_size` due to paddings, with the former being the shape of the model's logits. To be safe, always pass in the former when instantiating `xgr.TokenizerInfo`. ```python # Get tokenizer info model_id = "meta-llama/Llama-3.2-1B-Instruct" tokenizer = AutoTokenizer.from_pretrained(model_id) config = AutoConfig.from_pretrained(model_id) # This can be larger than tokenizer.vocab_size due to paddings full_vocab_size = config.vocab_size tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=full_vocab_size) ``` ### xgr.GrammarCompiler With an `xgr.TokenizerInfo`, we can instantiate an `xgr.GrammarCompiler`. This is a construct that compiles a grammar according to the model's tokenizer info. Therefore, for each model, you can use the same `xgr.GrammarCompiler` persistently, as it can compile different grammars for the same `xgr.TokenizerInfo`. Note that the `compiler` behavior can be configured with `max_threads` for multithreading, `cache_enabled` (defaults to true) for caching compiled grammars, and `cache_limit_bytes` for limiting the cache size. ```python compiler = xgr.GrammarCompiler(tokenizer_info, max_threads=8) ``` ### xgr.CompiledGrammar Then, using the `xgr.GrammarCompiler`, we can compile a grammar, with the result being an `xgr.CompiledGrammar`. Here we use a built-in JSON grammar. For other grammars, see [JSON Generation](json_generation.md) and [EBNF-Guided Generation](ebnf_guided_generation.md). Every thing we have seen up to now are per-model (rather than per-generation). ```python compiled_grammar: xgr.CompiledGrammar = compiler.compile_builtin_json_grammar() ``` ### xgr.GrammarMatcher With the compiled grammar, we can instantiate a `xgr.GrammarMatcher`. It is the main construct an LLM engine interacts with that maintains the state of the structured generation. Note that each request should have its own `xgr.GrammarMatcher` since each has a different generation state, as we will see in [Structured Generation for Batched Inference](#structured-generation-for-batched-inference). ```python # Instantiate grammar matcher with the compiled grammar matcher = xgr.GrammarMatcher(compiled_grammar) ``` ### Bitmasking Logits in Auto-regressive Generation Now we simulate a single-request auto-regressive generation. See later section for [Structured Generation for Batched Inference](#structured-generation-for-batched-inference). First, we pre-allocate a token bitmask with `xgr.allocate_token_bitmask()`. The bitmask compresses one bit per token into int32 storage, so it is a `torch.Tensor` of dtype `int32` and shape `(batch_size, ceil(vocab_size / 32))`. You can also use your own implementation for allocating a bitmask. In each auto-regressive step, we fill the token bitmask according to the current state of the matcher with `xgr.GrammarMatcher.fill_next_token_bitmask()`. Then, we apply the bitmask into the model's logits with `xgr.apply_token_bitmask_inplace()`, which calls a CUDA kernel if `logits` is on CUDA (recommended), otherwise a CPU implementation. After masking, the logits for illegal tokens are set to negative infinity, so that we will never sample them. After sampling the token, update the `xgr.GrammarMatcher`'s state with `xgr.GrammarMatcher.accept_token()`. Finally, use `xgr.GrammarMatcher.reset()` to prepare for the next generation. ```python # Here we simulate a valid sampled response sim_sampled_response = '{ "library": "xgrammar" }<|end_of_text|>' sim_sampled_token_ids = tokenizer.encode(sim_sampled_response, add_special_tokens=False) # Allocate a token bitmask token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) # Each loop iteration is a simulated auto-regressive step for i, sim_token_id in enumerate(sim_sampled_token_ids): # LLM inference to get logits, here we use randn to simulate. # logits is a tensor of shape (full_vocab_size,) on GPU # logits = LLM.inference() logits = torch.randn(full_vocab_size).cuda() # Apply bitmask to logits to mask invalid tokens matcher.fill_next_token_bitmask(token_bitmask) xgr.apply_token_bitmask_inplace(logits, token_bitmask.to(logits.device)) # Sample next token probs = torch.softmax(logits, dim=-1).cpu().numpy() next_token_id = np.random.choice(list(range(full_vocab_size)), p=probs) # Accept token from matcher to update its state, so that the next bitmask # generated will enforce the next token to be generated. Assert to make # sure the token is indeed valid. Here we accept the simulated response # assert matcher.accept_token(next_token_id) assert matcher.accept_token(sim_token_id) # Since we accepted a stop token `<|end_of_text|>`, we have terminated assert matcher.is_terminated() # Reset to be ready for the next auto-regressive generation matcher.reset() ``` ## Structured Generation for Batched Inference The code snippets above assume a single request generation. This section demonstrates how the same concept works with batched generation. First, follow the exact same steps above for the per-model constructs `xgr.TokenizerInfo` and `xgr.GrammarCompiler`. Say each request needs to generate a valid JSON. ```python import xgrammar as xgr import torch import numpy as np from transformers import AutoTokenizer, AutoConfig # Get tokenizer info model_id = "meta-llama/Llama-3.2-1B-Instruct" tokenizer = AutoTokenizer.from_pretrained(model_id) config = AutoConfig.from_pretrained(model_id) # This can be larger than tokenizer.vocab_size due to paddings full_vocab_size = config.vocab_size tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=full_vocab_size) # Compile a JSON grammar compiler = xgr.GrammarCompiler(tokenizer_info, max_threads=8) compiled_grammar: xgr.CompiledGrammar = compiler.compile_builtin_json_grammar() ``` Now, we need to maintain an `xgr.GrammarMatcher` for each request in the batch, since each has a different generation state. Note that each request in the batch can follow a different `xgr.CompiledGrammar`, but here for simplicity, they are all just following the general JSON grammar. ```python batch_size = 2 matchers = [ xgr.GrammarMatcher(compiled_grammar) for i in range(batch_size) ] token_bitmask = xgr.allocate_token_bitmask(batch_size, tokenizer_info.vocab_size) ``` We simulate an auto-regressive generation of batched inference. Note that here we assume the generation lengths of the two requests are the same for simplicity. But it should be easy to generalize based on how your engine supports batched inference. The key difference from single-request generation is that, in batched-request generation, each request has its own `xgr.GrammarMatcher` to maintain. ```python sim_sampled_responses = ['{"name": "a"}<|end_of_text|>', '{"name": "b"}<|end_of_text|>'] sim_sampled_token_ids = [ tokenizer.encode(response, add_special_tokens=False) for response in sim_sampled_responses ] # Each loop iteration is a simulated auto-regressive step for loop_iter in range(len(sim_sampled_token_ids[0])): # LLM batched inference to get logits, here we use randn to simulate # Now, logits is a tensor of shape (batch_size, full_vocab_size) on GPU # logits = LLM.inference() logits = torch.randn(batch_size, full_vocab_size).cuda() # This for loop is parallelizable using threading.Thread. But estimate # the overhead in your engine. for i in range(batch_size): matchers[i].fill_next_token_bitmask(token_bitmask, i) xgr.apply_token_bitmask_inplace(logits, token_bitmask.to(logits.device)) # Sample next token probs = torch.softmax(logits, dim=-1).cpu().numpy() next_token_ids = [ np.random.choice(list(range(full_vocab_size)), p=probs[i]) for i in range(batch_size) ] # Update the matcher for each request for i in range(batch_size): # Here we accept the simulated response # assert matchers[i].accept_token(next_token_ids[i]) matchers[i].accept_token(sim_sampled_token_ids[i][loop_iter]) # In our simulated case, all requests should have terminated since we accepted # a stop token `<|end_of_text|>` for i in range(batch_size): assert matchers[i].is_terminated() # Reset to be ready for the next generation matchers[i].reset() ``` xgrammar-0.2.3/docs/tutorials/json_generation.md000066400000000000000000000146631521764210300220310ustar00rootroot00000000000000# JSON Generation XGrammar enables efficient structured generation. One example structure is JSON and JSON Schema. In this tutorial, we go over how to use XGrammar to ensure that an LLM's output is a valid JSON, or adheres to a customized JSON schema. We first go over how to use XGrammar in an LLM engine to achieve this in [JSON Generation in LLM Engines](#json-generation-in-llm-engines), we then provide an end-to-end JSON generation using XGrammar with HF `transformers` in [Try out via HF Transformers](#try-out-via-hf-transformers). ## Install XGrammar [XGrammar](../start/installation) is available via pip. It is always recommended to install it in an isolated conda virtual environment. ## JSON Generation in LLM Engines In this section, we see how to use XGrammar in an LLM engine to ensure that the output is always a valid JSON. All code snippets below are actual runnable code as we simulate the LLM generation. First, import necessary libraries for the tutorial. ```python import xgrammar as xgr import torch import numpy as np from transformers import AutoTokenizer, AutoConfig ``` Then, we extract tokenizer info from the LLM we are using with `xgr.TokenizerInfo`. With the `tokenizer_info`, instantiate `xgr.GrammarCompiler` that will compiler a grammar of your choice. ```python # Get tokenizer info model_id = "meta-llama/Llama-3.2-1B-Instruct" tokenizer = AutoTokenizer.from_pretrained(model_id) config = AutoConfig.from_pretrained(model_id) # This can be larger than tokenizer.vocab_size due to paddings full_vocab_size = config.vocab_size tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=full_vocab_size) compiler = xgr.GrammarCompiler(tokenizer_info, max_threads=8) ``` For JSON generation, there are generally three options for compiling the grammar: using a built-in JSON grammar, specify JSON schema with a Pydantic model, or from a JSON schema string. Pick one of the three below to run. ```python # Option 1: Compile with a built-in JSON grammar compiled_grammar: xgr.CompiledGrammar = compiler.compile_builtin_json_grammar() ``` ```python # Option 2: Compile with JSON schema from a pydantic model from pydantic import BaseModel class Person(BaseModel): name: str age: int compiled_grammar = compiler.compile_json_schema(Person) ``` ```python # Option 3: Compile with JSON schema from a JSON schema string import json person_schema = { "title": "Person", "type": "object", "properties": { "name": { "type": "string" }, "age": { "type": "integer", } }, "required": ["name", "age"] } compiled_grammar = compiler.compile_json_schema(json.dumps(person_schema)) ``` With the compiled grammar, we can instantiate a `xgr.GrammarMatcher`, the main construct we interact with that maintains the state of the structured generation. We also allocate a bitmask that will be used to mask logits. ```python # Instantiate grammar matcher and allocate the bitmask matcher = xgr.GrammarMatcher(compiled_grammar) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) ``` Now we simulate a single-request auto-regressive generation. See [Integration with LLM Engine](engine_integration.md) for batched inference. ```python # Here we simulate a valid sampled response sim_sampled_response = '{"name": "xgrammar", "age": 0}<|end_of_text|>' sim_sampled_token_ids = tokenizer.encode(sim_sampled_response, add_special_tokens=False) # Each loop iteration is a simulated auto-regressive step for i, sim_token_id in enumerate(sim_sampled_token_ids): # LLM inference to get logits, here we use randn to simulate. # logits is a tensor of shape (full_vocab_size,) on GPU # logits = LLM.inference() logits = torch.randn(full_vocab_size).cuda() # Apply bitmask to logits to mask invalid tokens matcher.fill_next_token_bitmask(token_bitmask) xgr.apply_token_bitmask_inplace(logits, token_bitmask.to(logits.device)) # Sample next token probs = torch.softmax(logits, dim=-1).cpu().numpy() next_token_id = np.random.choice(list(range(full_vocab_size)), p=probs) # Accept token from matcher to update its state, so that the next bitmask # generated will enforce the next token to be generated. Assert to make # sure the token is indeed valid. Here we accept the simulated response # assert matcher.accept_token(next_token_id) assert matcher.accept_token(sim_token_id) # Since we accepted a stop token `<|end_of_text|>`, we have terminated assert matcher.is_terminated() # Reset to be ready for the next auto-regressive generation matcher.reset() ``` ## Try out via HF Transformers XGrammar can be easily integrated with HF transformers using a `LogitsProcessor`. Note that this integration mainly aims for accessibility and may contain extra overhead. First, instantiate a model, a tokenizer, and inputs. ```python import xgrammar as xgr import torch from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig device = "cuda" # Or "cpu", etc. model_name = "meta-llama/Llama-3.2-1B-Instruct" model = AutoModelForCausalLM.from_pretrained( model_name, torch_dtype=torch.float32, device_map=device ) tokenizer = AutoTokenizer.from_pretrained(model_name) config = AutoConfig.from_pretrained(model_name) messages = [ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Introduce yourself in JSON with two fields: name and age."}, ] texts = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) model_inputs = tokenizer(texts, return_tensors="pt").to(model.device) ``` Then construct a `GrammarCompiler` and compile the grammar. ```python tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=config.vocab_size) grammar_compiler = xgr.GrammarCompiler(tokenizer_info) # Option 1: Compile with a built-in JSON grammar # compiled_grammar = grammar_compiler.compile_builtin_json_grammar() # Option 2: Compile with JSON schema from a pydantic model from pydantic import BaseModel class Person(BaseModel): name: str age: int compiled_grammar = grammar_compiler.compile_json_schema(Person) ``` Finally, use `LogitsProcessor` to generate with grammar. ```python xgr_logits_processor = xgr.contrib.hf.LogitsProcessor(compiled_grammar) generated_ids = model.generate( **model_inputs, max_new_tokens=512, logits_processor=[xgr_logits_processor] ) generated_ids = generated_ids[0][len(model_inputs.input_ids[0]) :] print(tokenizer.decode(generated_ids, skip_special_tokens=True)) ``` xgrammar-0.2.3/docs/tutorials/workflow_of_xgrammar.md000066400000000000000000000166771521764210300231100ustar00rootroot00000000000000# Workflow of XGrammar This tutorial introduces the workflow of XGrammar, including most of its core components. Please read [constrained decoding](constrained_decoding.md) first to understand how XGrammar achieves structured generation. ```python import xgrammar as xgr import asyncio import torch from transformers import AutoTokenizer, AutoModelForCausalLM ``` ## Grammar [`xgr.Grammar`](xgrammar.Grammar) describes the structure of the LLM output. It can be: * A JSON schema or free-form JSON * A regex * A customized context-free grammar in the extended BNF format * etc. To construct a grammar, use ```python grammar: xgr.Grammar = xgr.Grammar.from_json_schema(json_schema_string) # or grammar: xgr.Grammar = xgr.Grammar.builtin_json_grammar() # or grammar: xgr.Grammar = xgr.Grammar.from_regex(regex_string) # or grammar: xgr.Grammar = xgr.Grammar.from_ebnf(ebnf_string) print(grammar) # print the ebnf format of the grammar ``` ## Tokenizer Info [`xgr.TokenizerInfo`](xgrammar.TokenizerInfo) contains the tokenizer information of the model. It is necessary for XGrammar to generate the token mask. [`xgr.TokenizerInfo`](xgrammar.TokenizerInfo) can be constructed from a HuggingFace tokenizer, or from a list of raw tokens. For HuggingFace tokenizers, XGrammar supports [HuggingFace's fast tokenizer](https://github.com/huggingface/tokenizers), [tiktoken](https://github.com/openai/tiktoken), and [SentencePiece](https://github.com/google/sentencepiece) tokenizers as the backend. ```python tokenizer = AutoTokenizer.from_pretrained(...) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) ``` If the `vocab_size` parameter is not provided, it defaults to the tokenizer's vocabulary size. Note that the model's vocabulary size (`config.vocab_size`, i.e. the size of its logits) can differ from the tokenizer's due to padding. In that case, pass the model's vocabulary size explicitly: `xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=config.vocab_size)`. See [integration with LLM engine](engine_integration.md) for details. ## Grammar Compiler To accelerate mask generation, XGrammar performs preprocessing on the grammar using the vocabulary of the model. This process is called **Grammar Compilation**. During grammar compilation, we: * Simplify the grammar and build automata * Compute an adaptive token mask cache. It will be used at runtime to generate the real mask [`xgr.GrammarCompiler`](xgrammar.GrammarCompiler) processes the grammar and produces a [`xgr.CompiledGrammar`](xgrammar.CompiledGrammar) object. Each [`xgr.GrammarCompiler`](xgrammar.GrammarCompiler) is bound to a specific [`xgr.TokenizerInfo`](xgrammar.TokenizerInfo) object. When given a grammar, it uses this tokenizer info to compile it. You can pass in a Grammar object directly, or provide a raw EBNF string, JSON Schema, or regex pattern: ```python grammar_compiler = xgr.GrammarCompiler(tokenizer_info) compiled_grammar = grammar_compiler.compile_grammar(grammar) # or compiled_grammar = grammar_compiler.compile_json_schema(json_schema_string) # or compiled_grammar = grammar_compiler.compile_builtin_json_grammar() # or compiled_grammar = grammar_compiler.compile_regex(regex_string) # or compiled_grammar = grammar_compiler.compile_grammar(ebnf_string) ``` ## Compiled Grammar A [`xgr.CompiledGrammar`](xgrammar.CompiledGrammar) object is associated with an [`xgr.Grammar`](xgrammar.Grammar) object and an [`xgr.TokenizerInfo`](xgrammar.TokenizerInfo) object. It contains the compiled grammar and the token mask cache. Use these methods to access the grammar and tokenizer info: ```python compiled_grammar.grammar compiled_grammar.tokenizer_info ``` ## Token Bitmask The mask is a bool tensor with the same shape as the vocabulary size. XGrammar further compresses the mask into a int32 bitset to save memory. It also support batch settings. Use [`xgr.allocate_token_bitmask`](xgrammar.allocate_token_bitmask) to allocate a bitmask: ```python bitmask = xgr.allocate_token_bitmask(batch_size, tokenizer_info.vocab_size) ``` The bitmask is a [`torch.Tensor`](https://pytorch.org/docs/stable/tensors.html) with the shape `(batch_size, ceil(vocab_size / 32))`, dtype `int32` and device `cpu`. It is located on CPU because we will further fill the bitmask with CPU logic. ## Grammar Matcher [`xgr.GrammarMatcher`](xgrammar.GrammarMatcher) handles the logic of matching the LLM output to the structure and generating the token mask. It is constructed with a [`xgr.CompiledGrammar`](xgrammar.CompiledGrammar) object. In each step, it will accept the last token generated by the LLM with [`xgr.GrammarMatcher.accept_token`](xgrammar.GrammarMatcher.accept_token), and then generate the mask with [`xgr.GrammarMatcher.fill_next_token_bitmask`](xgrammar.GrammarMatcher.fill_next_token_bitmask). ```python grammar_matcher = xgr.GrammarMatcher(compiled_grammar) token_id = ... # the last token generated by the LLM grammar_matcher.accept_token(token_id) grammar_matcher.fill_next_token_bitmask(bitmask) ``` Use [`xgr.apply_token_bitmask_inplace`](xgrammar.apply_token_bitmask_inplace) to apply the bitmask to the logits of the LLM. It will modify the logits in place. If the logits is on GPU, the bitmask should be moved to the same device first. ```python logits = ... # the logits of the LLM xgr.apply_token_bitmask_inplace(logits, bitmask.to(logits.device)) # Sample the next token prob = torch.softmax(logits, dim=-1) next_token_id = torch.argmax(prob, dim=-1) print(tokenizer.decode(next_token_id)) ``` ## The Generation Loop With the above introduction, it’s easy for us to write a generation loop using Hugging Face Transformers. [`xgr.GrammarMatcher.is_terminated`](xgrammar.GrammarMatcher.is_terminated) is provided to check if the matcher is terminated. ```python input_ids: list(int) = ... # the input tokens token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) model = AutoModelForCausalLM.from_pretrained(...) while not grammar_matcher.is_terminated(): # Generate the logits. Shape: (1, seq_len, vocab_size) logits = model(torch.tensor([input_ids], device=torch.device("cuda"))).logits # Fill and apply the bitmask grammar_matcher.fill_next_token_bitmask(token_bitmask) xgr.apply_token_bitmask_inplace(logits[0, -1, :], token_bitmask.to(logits.device)) # Sample the next token prob = torch.softmax(logits[0, -1, :], dim=-1) next_token_id = torch.argmax(prob, dim=-1) # Accept the token and append it to the input grammar_matcher.accept_token(next_token_id.item()) input_ids.append(next_token_id.item()) # Reset the matcher so it can be used again grammar_matcher.reset() # Print the generated text print(tokenizer.decode(input_ids)) ``` By default, [`xgr.GrammarMatcher.is_terminated`](xgrammar.GrammarMatcher.is_terminated) returns True after the matcher accepts a stop token following the completion of the structure. The stop tokens default to the ones detected from the tokenizer (e.g. the EOS token), and can be customized with the `override_stop_tokens` parameter of [`xgr.GrammarMatcher`](xgrammar.GrammarMatcher). Alternatively, if the matcher is constructed with `terminate_without_stop_token=True`, it terminates as soon as the structure is completed, without requiring a stop token. Congratulations! You have successfully generated a structured output using XGrammar. ## Next Steps Read [advanced topics](advanced_topics.md) to learn more advanced features about XGrammar. Read [integration with LLM engine](engine_integration.md) to learn how to integrate XGrammar into an LLM engine. xgrammar-0.2.3/docs/wrap_run_llm.py000066400000000000000000000034011521764210300173340ustar00rootroot00000000000000""" HTML post-processing script to insert RunLLM widget into documentation. Based on: https://github.com/sgl-project/sglang/blob/499f5e620c243b6a9980b63f7aa54d096a9a3ddd/docs/wrap_run_llm.py Copyright (c) 2023 SGLang Project (Apache 2.0 License) """ import os import re def insert_runllm_widget(html_content): # RunLLM Widget script to be inserted widget_script = """ """ # Find the closing body tag and insert the widget script before it return re.sub(r"", f"{widget_script}\n", html_content) def process_html_files(build_dir): for root, dirs, files in os.walk(build_dir): for file in files: if file.endswith(".html"): file_path = os.path.join(root, file) # Read the HTML file with open(file_path, "r", encoding="utf-8") as f: content = f.read() # Insert the RunLLM widget modified_content = insert_runllm_widget(content) # Write back the modified content with open(file_path, "w", encoding="utf-8") as f: f.write(modified_content) def main(): # Get the build directory path build_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "_build", "html") # Process all HTML files if os.path.exists(build_dir): process_html_files(build_dir) else: print(f"Build directory not found: {build_dir}") if __name__ == "__main__": main() xgrammar-0.2.3/docs/xgrammar_features/000077500000000000000000000000001521764210300177775ustar00rootroot00000000000000xgrammar-0.2.3/docs/xgrammar_features/javascript_api.md000066400000000000000000000007231521764210300233220ustar00rootroot00000000000000# JavaScript API Beside the Python and C++ API, XGrammar also provides JavaScript/TypeScript API. The JS SDK uses [emscripten](https://emscripten.org/) to compile the C++ code into WebAssembly. It is designed to be used for LLMs that run in the browser, such as [WebLLM](https://github.com/mlc-ai/web-llm). To use this SDK, run: ```bash npm install @mlc-ai/web-xgrammar ``` For more information, see [the code](https://github.com/mlc-ai/xgrammar/tree/main/web). xgrammar-0.2.3/docs/xgrammar_features/runtime_safeguards.md000066400000000000000000000040051521764210300242070ustar00rootroot00000000000000# Runtime Safeguards XGrammar has a set of mechanisms to safeguard the runtime and avoid the LLM server from crashing. ## Recursion Limit The [`xgr.GrammarMatcher`](xgrammar.GrammarMatcher) class uses a pushdown automata parser to parse the grammar. It may involve very deep recursion, which may cause stack overflow. XGrammar provides [`xgr.set_max_recursion_depth`](xgrammar.set_max_recursion_depth) to set the maximum recursion depth and [`xgr.get_max_recursion_depth`](xgrammar.get_max_recursion_depth) to get the current maximum recursion depth. The maximum recursion depth is set per process.The default maximum recursion depth is 10000. If the recursion depth exceeds the limit, the matcher operations (including [`xgr.GrammarMatcher.accept_token`](xgrammar.GrammarMatcher.accept_token), [`xgr.GrammarMatcher.accept_string`](xgrammar.GrammarMatcher.accept_string), [`xgr.GrammarMatcher.fill_next_token_bitmask`](xgrammar.GrammarMatcher.fill_next_token_bitmask), [`xgr.GrammarMatcher.find_jump_forward_string`](xgrammar.GrammarMatcher.find_jump_forward_string)) will raise `RuntimeError`(before XGrammar v0.1.21). You can also use the [`xgr.max_recursion_depth`](xgrammar.max_recursion_depth) context manager to set the maximum recursion depth for a code block. ```python from xgrammar import max_recursion_depth with max_recursion_depth(10000): matcher.accept_token(token_id) ``` After XGrammar v0.1.21, the pushdown automaton parser was replaced with an Earley parser, and there is no recursion involved. So the recursion will never be exceed during parsing, and the exception will not be raised. ## Cache Size Limit The {py:class}`xgr.GrammarCompiler` class uses a cache to store the compiled grammars. The cache size can be limited to avoid the cache from growing too large. The cache uses an LRU algorithm to evict the least recently used items. The cache size limit is -1 by default, which means no limit. ```python from xgrammar import GrammarCompiler compiler = GrammarCompiler(tokenizer_info, cache_limit_bytes=10000) ``` xgrammar-0.2.3/docs/xgrammar_features/serialization.md000066400000000000000000000075731521764210300232120ustar00rootroot00000000000000# Serialization XGrammar supports serialization and deserialization for caching and cross-process and inter-server communication purposes. We currently support serialization to and deserialization from JSON. The classes that support serialization and deserialization are introduced below. Each class has a `serialize_json` method to serialize the type to a JSON string, and a `deserialize_json` method to deserialize the type from a JSON string. Each serialized result have a `__VERSION__` field to indicate the serialization version. When the internal data structure is changed in XGrammar, the serialization version will be updated. Use [`xgr.get_serialization_version`](xgrammar.get_serialization_version) to get the current serialization version. In deserialization, if the version in the JSON string does not match the current version, the deserialization will fail and raise a [`xgr.exception.DeserializeVersionError`](xgrammar.exception.DeserializeVersionError). > **Note:**
> After upgrading XGrammar, the serialization format may change. If you have cached serialization > results to disk, please clear the cache after the upgrade to avoid potential version conflicts. Three error types are raised when deserialization fails: - [`xgr.exception.InvalidJSONError`](xgrammar.exception.InvalidJSONError): When the JSON string is invalid. - [`xgr.exception.DeserializeFormatError`](xgrammar.exception.DeserializeFormatError): When the JSON string does not follow the serialization format of the type. - [`xgr.exception.DeserializeVersionError`](xgrammar.exception.DeserializeVersionError): When the serialization version in the JSON string is not the same as the current version. ## [`xgr.Grammar`](xgrammar.Grammar) The grammar class. ```python import xgrammar as xgr from transformers import AutoTokenizer # Construct Grammar grammar: xgr.Grammar = xgr.Grammar.builtin_json_grammar() # Serialize to JSON grammar_json: str = grammar.serialize_json() print(f"Serialized Grammar: {grammar_json}") # Deserialize from JSON grammar_deserialized: xgr.Grammar = xgr.Grammar.deserialize_json(grammar_json) print(f"Deserialized Grammar: {grammar_deserialized}") ``` ## [`xgr.TokenizerInfo`](xgrammar.TokenizerInfo) The tokenizer information class. ```python # Construct TokenizerInfo tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-3.1-8B-Instruct") tokenizer_info: xgr.TokenizerInfo = xgr.TokenizerInfo.from_huggingface(tokenizer) # Serialize to JSON tokenizer_info_json: str = tokenizer_info.serialize_json() print(f"Serialized TokenizerInfo") # Deserialize from JSON tokenizer_info_deserialized: xgr.TokenizerInfo = xgr.TokenizerInfo.deserialize_json( tokenizer_info_json ) print(f"Deserialized TokenizerInfo") ``` ## [`xgr.CompiledGrammar`](xgrammar.CompiledGrammar) A `CompiledGrammar` contains a tokenizer info, and multiple `CompiledGrammar`s can share the same `TokenizerInfo` to avoid redundant storage. So in serialization of a `CompiledGrammar`, we will not include the complete `TokenizerInfo`, but only the metadata of it. During deserialization, we will require a `TokenizerInfo` object and check if it matches the metadata in the serialized JSON string. If so, the deserialization will use the provided `TokenizerInfo` object in the result. If not, the deserialization will raise a [`xgr.exception.DeserializeFormatError`](xgrammar.exception.DeserializeFormatError). ```python # Construct CompiledGrammar compiler: xgr.GrammarCompiler = xgr.GrammarCompiler(tokenizer_info_deserialized) compiled_grammar: xgr.CompiledGrammar = compiler.compile_grammar(grammar_deserialized) # Serialize to JSON compiled_grammar_json: str = compiled_grammar.serialize_json() print(f"Serialized CompiledGrammar") # Deserialize from JSON compiled_grammar_deserialized: xgr.CompiledGrammar = xgr.CompiledGrammar.deserialize_json( compiled_grammar_json, tokenizer_info_deserialized ) print(f"Deserialized CompiledGrammar") ``` xgrammar-0.2.3/examples/000077500000000000000000000000001521764210300151515ustar00rootroot00000000000000xgrammar-0.2.3/examples/benchmark/000077500000000000000000000000001521764210300171035ustar00rootroot00000000000000xgrammar-0.2.3/examples/benchmark/README.md000066400000000000000000000042571521764210300203720ustar00rootroot00000000000000 ## Run Benchmark ### Benchmark Grammar Compile and Mask Generation #### Dependencies ``` outlines 0.1.3 outlines_core 0.1.14 lm-format-enforcer 0.10.6 ``` #### Run ```bash python3 bench_grammar_compile_mask_gen.py [-h] [--backend {xgrammar,outlines,lmformatenforcer}] [--num_iters NUM_ITERS] [--num_warmup NUM_WARMUP] ``` ### Benchmark Apply Token Bitmask Inplace Kernels #### Run ```bash python3 examples/benchmark/bench_apply_token_bitmask_inplace.py ``` #### Results H100 | Batch | Vocab | Masked cnt | Torch Compile | Triton | CUDA | | size | size | | Baseline us | us (speedup) | us (speedup) | |--------:|--------:|-------------:|----------------:|----------------:|----------------:| | 1 | 128000 | 1 | 5.85 | 5.41 (1.08x) | 5.46 (1.07x) | | 1 | 128000 | 64000 | 5.84 | 6.01 (0.97x) | 6.24 (0.94x) | | 1 | 128000 | 127000 | 5.84 | 6.09 (0.96x) | 5.95 (0.98x) | | 8 | 128000 | 1 | 10.75 | 5.86 (1.83x) | 5.90 (1.82x) | | 8 | 128000 | 64000 | 10.75 | 7.59 (1.42x) | 9.85 (1.09x) | | 8 | 128000 | 127000 | 10.77 | 7.85 (1.37x) | 8.06 (1.34x) | | 64 | 128000 | 1 | 48.59 | 13.10 (3.71x) | 9.68 (5.02x) | | 64 | 128000 | 64000 | 48.59 | 45.43 (1.07x) | 38.76 (1.25x) | | 64 | 128000 | 127000 | 48.58 | 32.84 (1.48x) | 26.29 (1.85x) | | 512 | 128000 | 1 | 349.84 | 67.34 (5.20x) | 37.06 (9.44x) | | 512 | 128000 | 64000 | 346.94 | 330.36 (1.05x) | 256.53 (1.35x) | | 512 | 128000 | 127000 | 345.54 | 249.66 (1.38x) | 157.51 (2.19x) | | 4096 | 128000 | 1 | 2895.83 | 494.47 (5.86x) | 249.96 (11.59x) | | 4096 | 128000 | 64000 | 2863.31 | 2517.85 (1.14x) | 1993.29 (1.44x) | | 4096 | 128000 | 127000 | 2720.67 | 1935.24 (1.41x) | 1207.38 (2.25x) | xgrammar-0.2.3/examples/benchmark/bench_apply_token_bitmask_inplace.py000066400000000000000000000121221521764210300263440ustar00rootroot00000000000000# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import argparse from itertools import product from typing import Any import torch from tabulate import tabulate from tqdm import tqdm from triton.testing import do_bench from xgrammar.kernels.apply_token_bitmask_inplace_cuda import apply_token_bitmask_inplace_cuda from xgrammar.kernels.apply_token_bitmask_inplace_torch_compile import ( apply_token_bitmask_inplace_torch_compile, ) from xgrammar.kernels.apply_token_bitmask_inplace_triton import apply_token_bitmask_inplace_triton from xgrammar.testing import _bool_mask_to_bitmask IMPL_TORCH_COMPILE: str = "Torch Compile" IMPL_TRITON: str = "Triton" IMPL_CUDA: str = "CUDA" ALL_IMPLS: list[str] = [IMPL_TORCH_COMPILE, IMPL_TRITON, IMPL_CUDA] def bench_single_impl( impl: str, logits: torch.Tensor, bitmask: torch.Tensor, logits_expected: torch.Tensor, kwargs: dict[str, Any], args: argparse.Namespace, ) -> float: if impl == IMPL_TORCH_COMPILE: f = lambda: apply_token_bitmask_inplace_torch_compile(logits, bitmask, **kwargs) elif impl == IMPL_TRITON: f = lambda: apply_token_bitmask_inplace_triton(logits, bitmask, **kwargs) else: f = lambda: apply_token_bitmask_inplace_cuda(logits, bitmask, **kwargs) f() torch.testing.assert_close(logits, logits_expected.to("cuda")) torch.cuda.synchronize() exec_time = do_bench(f, warmup=args.warmup, rep=args.rep) return exec_time * 1000 def bench_single_setup(batch_size: int, masked_cnt: int, args: argparse.Namespace) -> list[float]: vocab_size = args.vocab_size stride = args.stride logits_dtype = getattr(torch, args.logits_dtype) logits = torch.randn(batch_size, vocab_size, dtype=logits_dtype, device="cuda") if masked_cnt >= vocab_size: bool_mask = torch.zeros(batch_size, vocab_size, dtype=torch.bool, device="cuda") else: bool_mask = torch.ones(batch_size, vocab_size, dtype=torch.bool, device="cuda") if masked_cnt > 0: masked_positions = torch.stack( [torch.randperm(vocab_size, device="cuda")[:masked_cnt] for _ in range(batch_size)] ) bool_mask.scatter_(1, masked_positions, False) assert (bool_mask.sum(dim=-1) + masked_cnt == vocab_size).all().item() bitmask = _bool_mask_to_bitmask(bool_mask) masked_batch_ids = torch.arange(0, batch_size, stride, dtype=torch.int32, device="cuda") kwargs = {} if stride == 1 else {"indices": masked_batch_ids} logits_copies = [logits.clone() for _ in range(len(args.impl))] logits[masked_batch_ids] = torch.masked_fill( logits[masked_batch_ids], ~bool_mask[masked_batch_ids], float("-inf") ) return [ bench_single_impl(impl, logits_copy, bitmask, logits, kwargs, args) for impl, logits_copy in zip(args.impl, logits_copies) ] if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "--impl", type=str, nargs="*", choices=ALL_IMPLS, default=[IMPL_TORCH_COMPILE, IMPL_TRITON, IMPL_CUDA], ) parser.add_argument("--batch-size", type=int, nargs="*", default=[1, 8, 64, 512, 4096]) parser.add_argument("--vocab-size", type=int, default=128000) parser.add_argument("--masked-cnt", type=int, nargs="*", default=[1, 64000, 127000]) parser.add_argument("--stride", type=int, default=1) parser.add_argument( "--logits_dtype", type=str, choices=["float32", "float16", "bfloat16"], default="float32" ) parser.add_argument("--warmup", type=int, default=500) parser.add_argument("--rep", type=int, default=2000) args = parser.parse_args() data_rows = [] for batch_size, masked_cnt in tqdm(list(product(args.batch_size, args.masked_cnt))): all_us = bench_single_setup(batch_size, masked_cnt, args) data_rows.append( [ batch_size, args.vocab_size, masked_cnt, f"{all_us[0]:.2f}", *[f"{us:.2f} ({all_us[0]/us:>4.2f}x)" for us in all_us[1:]], ] ) print( tabulate( data_rows, headers=[ "Batch\nsize", "Vocab\nsize", "Masked cnt", f"{args.impl[0]}\nBaseline us", *[f"{impl} \nus (speedup)" for impl in args.impl[1:]], ], tablefmt="pipe", floatfmt=".2f", colalign=["right"] * len(data_rows[0]), ) ) xgrammar-0.2.3/examples/benchmark/bench_grammar_compile_mask_gen.py000066400000000000000000000157241521764210300256270ustar00rootroot00000000000000"""This script benchmarks the time for grammar compilation and mask generation.""" import argparse import json import time import datasets import torch from lmformatenforcer import JsonSchemaParser, TokenEnforcer from lmformatenforcer.integrations.transformers import ( TokenEnforcerTokenizerData, build_token_enforcer_tokenizer_data, ) from outlines.fsm.guide import Guide, RegexGuide from outlines.fsm.json_schema import convert_json_schema_to_str from outlines.generate.generator import bias_logits from outlines.generate.json import build_regex_from_schema from outlines.models import TransformerTokenizer from tqdm import tqdm from transformers import AutoTokenizer import xgrammar as xgr wrong_data_indices = [1] def xgrammar_build(schema: str, grammar_compiler: xgr.GrammarCompiler): grammar = grammar_compiler.compile_json_schema(schema) matcher = xgr.GrammarMatcher(grammar) return matcher def xgrammar_exec( matcher: xgr.GrammarMatcher, logits: torch.Tensor, bitmask: torch.Tensor, token_id: int ): # Logits processing matcher.fill_next_token_bitmask(bitmask) xgr.apply_token_bitmask_inplace(logits, bitmask) # Update state assert matcher.accept_token(token_id) return def outlines_build(schema: str, tokenizer: TransformerTokenizer): schema_str = convert_json_schema_to_str(json_schema=schema) regex_string = build_regex_from_schema(schema_str, whitespace_pattern=None) guide = RegexGuide.from_regex(regex_string, tokenizer) return guide def outlines_exec(guide: Guide, logits: torch.Tensor, token_id: int, state=None): if state is None: state = guide.initial_state # Logits processing allowed_tokens = guide.get_next_instruction(state).tokens biased_logits = bias_logits(logits.view(1, -1), [allowed_tokens]) # Update state next_state = guide.get_next_state(state, token_id) return next_state def lmformatenforcer_build(schema: str, tokenizer: TokenEnforcerTokenizerData): parser = JsonSchemaParser(json.loads(schema)) token_enforcer = TokenEnforcer(tokenizer, parser) return token_enforcer def lmformatenforcer_exec(token_enforcer: TokenEnforcer, logits: torch.Tensor, token_ids): # Logits processing allowed_tokens = token_enforcer.get_allowed_tokens(token_ids) logits[allowed_tokens] = float("-inf") # Update state return if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "--backend", type=str, choices=["xgrammar", "outlines", "lmformatenforcer"], default="xgrammar", ) parser.add_argument("--num_iters", type=int, default=5) parser.add_argument("--num_warmup", type=int, default=-1) args = parser.parse_args() backend = args.backend num_iters = args.num_iters num_warmup = args.num_warmup if args.num_warmup != -1 else 5 if num_iters >= 40 else 1 dataset = datasets.load_dataset("NousResearch/json-mode-eval", split="train") hf_model_path = "meta-llama/Llama-3.1-8B-Instruct" hf_tokenizer = AutoTokenizer.from_pretrained(hf_model_path) xgrammar_tokenizer_info = xgr.TokenizerInfo.from_huggingface(hf_tokenizer) xgrammar_grammar_compiler = xgr.GrammarCompiler(xgrammar_tokenizer_info) outlines_tokenizer = TransformerTokenizer(hf_tokenizer) lmformatenforcer_tokenizer = build_token_enforcer_tokenizer_data(hf_tokenizer) vocab_size = len(hf_tokenizer) build_time = 0 exec_time = 0 total_data_points = 0 total_tokens = 0 fail_cnt = 0 tqdm_iter = tqdm(range(-num_warmup, num_iters)) for iter in tqdm_iter: if iter < 0: tqdm_iter.set_description(f"Backend: {backend}, Warmup Iter: {iter + num_warmup}") else: tqdm_iter.set_description(f"Backend: {backend}, Iter: {iter}") if iter == 0: # Reset time build_time = 0 exec_time = 0 tqdm_data_point_iter = tqdm(range(len(dataset))) for data_point_idx in tqdm_data_point_iter: tqdm_data_point_iter.set_description( f"Backend: {backend}, Data Point: {data_point_idx}" ) if data_point_idx in wrong_data_indices: continue schema = dataset["schema"][data_point_idx] completion = dataset["completion"][data_point_idx] token_ids = hf_tokenizer.encode(completion, add_special_tokens=False) prompt = hf_tokenizer.apply_chat_template( dataset["prompt"][data_point_idx], tokenize=False ) prompt_token_ids = hf_tokenizer.encode(prompt) print(f"Prompt: {prompt}, Schema: {schema}") start = time.perf_counter() try: if backend == "xgrammar": worker = xgrammar_build(schema, xgrammar_grammar_compiler) bitmask = xgr.allocate_token_bitmask(worker.vocab_size) elif backend == "outlines": worker = outlines_build(schema, outlines_tokenizer) elif backend == "lmformatenforcer": worker = lmformatenforcer_build(schema, lmformatenforcer_tokenizer) except Exception as e: if iter >= 0: fail_cnt += 1 continue build_time += time.perf_counter() - start # use different logits for each mask generation process # to avoid caching effects between different tokens logits = [torch.randn(vocab_size).cuda() for _ in range(len(token_ids))] torch.cuda.synchronize() start = time.perf_counter() fail_flag = False for idx, token_id in enumerate(token_ids): # Logits processing try: if backend == "xgrammar": xgrammar_exec(worker, logits[idx], bitmask, token_id) elif backend == "outlines": if idx == 0: state = None state = outlines_exec(worker, logits[idx], token_id, state) elif backend == "lmformatenforcer": lmformatenforcer_exec( worker, logits[idx], prompt_token_ids + token_ids[:idx] ) except Exception as e: if iter >= 0: fail_cnt += 1 fail_flag = True break if fail_flag: continue torch.cuda.synchronize() exec_time += time.perf_counter() - start if iter >= 0: total_data_points += 1 total_tokens += len(token_ids) print(f"Backend: {backend}") print(f"Fail count: {fail_cnt / num_iters:.0f} / {len(dataset) - len(wrong_data_indices)}") print(f"Grammar preprocessing time (ms): {build_time / total_data_points * 1e3:.4f}") print(f"Mask generation time (us/token): {exec_time / total_tokens * 1e6:.4f}") xgrammar-0.2.3/examples/benchmark/cibench_grammar_compile_mask_gen.py000066400000000000000000000366131521764210300261430ustar00rootroot00000000000000"""This script benchmarks the time for grammar compilation and mask generation using XGrammar.""" import argparse import json import time from typing import Any, Dict, List, Tuple import datasets import requests import torch from tqdm import tqdm from transformers import AutoTokenizer import xgrammar as xgr wrong_data_indices = [1] def xgrammar_build(schema: str, grammar_compiler: xgr.GrammarCompiler): grammar = grammar_compiler.compile_json_schema(schema) matcher = xgr.GrammarMatcher(grammar) return matcher def download_gorilla_file(filename: str) -> Tuple[List, List]: base_url = "https://raw.githubusercontent.com/ShishirPatil/gorilla/main/berkeley-function-call-leaderboard/data" function_url = f"{base_url}/{filename}" answer_url = f"{base_url}/possible_answer/{filename}" print(f"Downloading {filename} from GitHub...") try: function_response = requests.get(function_url) function_response.raise_for_status() function_text = function_response.text functions_data = [] for line in function_text.strip().split("\n"): if line.strip(): try: functions_data.append(json.loads(line)) except json.JSONDecodeError as e: print(f"Error parsing function line in {filename}: {e}") answer_response = requests.get(answer_url) answer_response.raise_for_status() answer_text = answer_response.text answers_data = [] for line in answer_text.strip().split("\n"): if line.strip(): try: answers_data.append(json.loads(line)) except json.JSONDecodeError as e: print(f"Error parsing answer line in {filename}: {e}") print( f"Successfully downloaded {filename}: {len(functions_data)} functions, {len(answers_data)} answers" ) return functions_data, answers_data except requests.RequestException as e: print(f"Error downloading {filename}: {e}") return [], [] def load_gorilla_data() -> List[Dict[str, Any]]: gorilla_data = [] # excluding live test cases part of BFCL v2/v3 file_patterns = [ "BFCL_v3_java.json", "BFCL_v3_javascript.json", "BFCL_v3_multiple.json", "BFCL_v3_parallel.json", "BFCL_v3_parallel_multiple.json", "BFCL_v3_simple.json", "BFCL_v3_sql.json", ] filtered_count = 0 for filename in file_patterns: functions_data, answers_data = download_gorilla_file(filename) if not functions_data or not answers_data: print(f"Skipping {filename} - failed to download data") continue print(f"Processing {filename}...") answers_by_id = {item["id"]: item for item in answers_data} for item in functions_data: item_id = item["id"] if item_id not in answers_by_id: print(f"Warning: No answer found for item {item_id}") continue if "function" not in item or not item["function"]: print(f"Warning: No function definition for item {item_id}") filtered_count += 1 continue if len(item["function"]) > 1: # print(f"Skipping item {item_id} - contains multiple functions ({len(item['function'])})") filtered_count += 1 continue function_def = item["function"][0] # Use the first function schema = convert_function_to_schema(function_def) answer = answers_by_id[item_id] if "ground_truth" not in answer or not answer["ground_truth"]: print(f"Warning: No ground truth for item {item_id}") filtered_count += 1 continue ground_truth = answer["ground_truth"][0] # Use the first ground truth completion = convert_ground_truth_to_completion(ground_truth) gorilla_data.append( {"schema": schema, "completion": completion, "id": item_id, "source": filename} ) print( f"Loaded {len(gorilla_data)} examples from Gorilla BFCL dataset (filtered out {filtered_count} examples)" ) return gorilla_data def convert_function_to_schema(function_def: Dict) -> str: """Convert a Gorilla function definition to a JSON schema string with improved type handling.""" function_name = function_def["name"] parameters = function_def["parameters"] schema = { "type": "object", "properties": {function_name: {"type": "object", "properties": {}, "required": []}}, "required": [function_name], } for key, value in parameters.get("properties", {}).items(): param_type = value.get("type", "string").lower() if param_type == "integer": schema_def = {"type": "integer"} elif param_type in ("float", "number", "double"): schema_def = {"type": "number"} elif param_type == "boolean": schema_def = {"type": "boolean"} elif param_type in ("hashmap", "map", "dict", "dictionary"): schema_def = {"type": "object", "additionalProperties": True} elif param_type in ("array", "list"): schema_def = {"type": "array", "items": {"type": "string"}} elif param_type == "any": schema_def = {} # No type restriction else: schema_def = {"type": "string"} schema["properties"][function_name]["properties"][key] = schema_def required_fields = parameters.get("required", []) if required_fields: schema["properties"][function_name]["required"] = required_fields return json.dumps(schema) def convert_ground_truth_to_completion(ground_truth: Dict) -> str: """Convert a Gorilla ground truth to a completion string with improved handling of nested structures.""" function_name = list(ground_truth.keys())[0] params = ground_truth[function_name] transformed_params = {} for key, values in params.items(): if isinstance(values, list) and len(values) == 1 and isinstance(values[0], dict): nested_obj = {} for nested_key, nested_values in values[0].items(): if isinstance(nested_values, list) and nested_values: nested_obj[nested_key] = nested_values[0] else: nested_obj[nested_key] = nested_values transformed_params[key] = nested_obj elif isinstance(values, list) and values: transformed_params[key] = values[0] else: transformed_params[key] = None completion = {function_name: transformed_params} return json.dumps(completion) def run_benchmark( dataset_name: str, dataset_data, tokenizer_info, hf_tokenizer, num_iters, num_warmup ): vocab_size = len(hf_tokenizer) build_time = 0 exec_time = 0 total_data_points = 0 total_tokens = 0 fail_cnt = 0 schema_mismatch_cnt = 0 tqdm_iter = tqdm(range(-num_warmup, num_iters), disable=True) for iter in tqdm_iter: if iter < 0: tqdm_iter.set_description(f"{dataset_name} Warmup Iter: {iter + num_warmup}") else: tqdm_iter.set_description(f"{dataset_name} Iter: {iter}") if iter == 0: build_time = 0 exec_time = 0 tqdm_data_point_iter = tqdm(range(len(dataset_data)), disable=True) for data_point_idx in tqdm_data_point_iter: tqdm_data_point_iter.set_description(f"{dataset_name} Data Point: {data_point_idx}") if dataset_name == "json-mode-eval" and data_point_idx in wrong_data_indices: continue schema = dataset_data[data_point_idx]["schema"] completion = dataset_data[data_point_idx]["completion"] if dataset_name == "gorilla-bfcl": try: schema_obj = json.loads(schema) completion_obj = ( json.loads(completion) if isinstance(completion, str) else completion ) schema_function_name = schema_obj.get("required", [""])[0] completion_function_name = ( list(completion_obj.keys())[0] if completion_obj else "" ) if ( schema_function_name and completion_function_name and schema_function_name != completion_function_name ): if iter >= 0: schema_mismatch_cnt += 1 if iter == 0: print( f"Schema-completion function name mismatch for data point {data_point_idx}:" ) print(f" Schema expects: {schema_function_name}") print(f" Completion has: {completion_function_name}") continue except Exception as e: # If there's an issue parsing the JSON, proceed anyway pass if isinstance(completion, dict): completion = json.dumps(completion) token_ids = hf_tokenizer.encode(completion, add_special_tokens=False) grammar_compiler = xgr.GrammarCompiler(tokenizer_info) start = time.perf_counter() try: worker = xgrammar_build(schema, grammar_compiler) bitmask = xgr.allocate_token_bitmask(1, vocab_size) except Exception as e: if iter >= 0: fail_cnt += 1 if iter == 0: print(f"Failed to build grammar for data point {data_point_idx}: {e}") continue build_time += time.perf_counter() - start # Use different logits for each mask generation process # to avoid caching effects between different tokens logits = [torch.randn(vocab_size).cuda() for _ in range(len(token_ids))] torch.cuda.synchronize() start = time.perf_counter() fail_flag = False token_rejection_count = 0 # give some leniency, can remove for idx, token_id in enumerate(token_ids): try: worker.fill_next_token_bitmask(bitmask) cuda_bitmask = bitmask.cuda() xgr.apply_token_bitmask_inplace(logits[idx], cuda_bitmask) # Update state if not worker.accept_token(token_id): token_rejection_count += 1 if token_rejection_count > 5: fail_flag = True break except Exception as e: if iter >= 0: if iter == 0: # Only print once to avoid spam print( f"Failed to process token {idx} for data point {data_point_idx}: {e}" ) fail_flag = True break if fail_flag: if iter >= 0: fail_cnt += 1 continue torch.cuda.synchronize() exec_time += time.perf_counter() - start if iter >= 0: total_data_points += 1 total_tokens += len(token_ids) results = { "dataset": dataset_name, "successful_data_points": total_data_points / num_iters if num_iters > 0 else 0, "failed_data_points": fail_cnt / num_iters if num_iters > 0 else 0, "schema_mismatch_count": schema_mismatch_cnt / num_iters if num_iters > 0 else 0, "total_possible_data_points": len(dataset_data) - (len(wrong_data_indices) if dataset_name == "json-mode-eval" else 0), "grammar_compilation_time_ms": ( build_time / total_data_points * 1e3 if total_data_points > 0 else float("inf") ), "per_token_overhead_us_per_token": ( exec_time / total_tokens * 1e6 if total_tokens > 0 else float("inf") ), } return results if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--num_iters", type=int, default=5) parser.add_argument("--num_warmup", type=int, default=-1) parser.add_argument( "--datasets", type=str, default="all", help="Datasets to benchmark: json-mode-eval, gorilla, or all", ) args = parser.parse_args() num_iters = args.num_iters num_warmup = args.num_warmup if args.num_warmup != -1 else 5 if num_iters >= 40 else 1 selected_datasets = args.datasets.lower() hf_model_path = "meta-llama/Llama-3.1-8B-Instruct" print(f"Loading tokenizer from {hf_model_path}...") hf_tokenizer = AutoTokenizer.from_pretrained(hf_model_path) xgrammar_tokenizer_info = xgr.TokenizerInfo.from_huggingface(hf_tokenizer) # Try to get GPU info try: device_count = torch.cuda.device_count() device_name = torch.cuda.get_device_name(0) if device_count > 0 else "No GPU" print(f"Running benchmark with: {device_name} (Device count: {device_count})") except Exception: print("Could not detect GPU information") results = [] if selected_datasets in ["json-mode-eval", "all"]: print("Loading json-mode-eval dataset...") json_mode_eval_dataset = datasets.load_dataset("NousResearch/json-mode-eval", split="train") json_mode_eval_data = [ {"schema": item["schema"], "completion": item["completion"]} for item in json_mode_eval_dataset ] print(f"Running benchmark on json-mode-eval ({len(json_mode_eval_data)} examples)...") json_mode_eval_results = run_benchmark( "json-mode-eval", json_mode_eval_data, xgrammar_tokenizer_info, hf_tokenizer, num_iters, num_warmup, ) results.append(json_mode_eval_results) if selected_datasets in ["gorilla", "all"]: print("Loading Gorilla BFCL dataset directly from GitHub...") gorilla_data = load_gorilla_data() if gorilla_data: print(f"Running benchmark on Gorilla BFCL ({len(gorilla_data)} examples)...") gorilla_results = run_benchmark( "gorilla-bfcl", gorilla_data, xgrammar_tokenizer_info, hf_tokenizer, num_iters, num_warmup, ) results.append(gorilla_results) else: print("No Gorilla data loaded, skipping benchmark") print("\n===== XGrammar Benchmark Results =====") print(f"Model: {hf_model_path}") print(f"Iterations: {num_iters}") print(f"Warmup Iterations: {num_warmup}") for result in results: print(f"\nDataset: {result['dataset']}") print( f"Successful data points: {result['successful_data_points']:.0f} / {result['total_possible_data_points']}" ) print( f"Failed data points: {result['failed_data_points']:.0f} / {result['total_possible_data_points']}" ) print(f"Grammar compilation time (ms): {result['grammar_compilation_time_ms']:.4f}") print(f"Per token overhead (us/token): {result['per_token_overhead_us_per_token']:.4f}") xgrammar-0.2.3/examples/hf_transformers/000077500000000000000000000000001521764210300203535ustar00rootroot00000000000000xgrammar-0.2.3/examples/hf_transformers/transformers_example.py000066400000000000000000000046231521764210300251720ustar00rootroot00000000000000""" This example demonstrates how to use XGrammar in Huggingface's transformers, integrated with a minimal LogitsProcessor. """ import torch from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer import xgrammar as xgr device = "cuda" # device = "cpu" # 0. Instantiate with any HF model you want model_name = "Qwen/Qwen2.5-0.5B-Instruct" # model_name = "microsoft/Phi-3.5-mini-instruct" # model_name = "meta-llama/Llama-3.2-1B-Instruct" model = AutoModelForCausalLM.from_pretrained( model_name, torch_dtype=torch.float32, device_map=device ) tokenizer = AutoTokenizer.from_pretrained(model_name) config = AutoConfig.from_pretrained(model_name) # This can be larger than tokenizer.vocab_size due to paddings full_vocab_size = config.vocab_size # 1. Compile grammar (NOTE: you can substitute this with other grammars like EBNF, JSON Schema) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=full_vocab_size) grammar_compiler = xgr.GrammarCompiler(tokenizer_info) compiled_grammar: xgr.CompiledGrammar = grammar_compiler.compile_builtin_json_grammar() # 2. Prepare inputs messages_list = [] prompts = [ "Introduce yourself in JSON briefly as a student.", # Uncomment for batch generation # "Introduce yourself in JSON as a professor.", ] for prompt in prompts: messages = [ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": prompt}, ] messages_list.append(messages) texts = [ tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) for messages in messages_list ] # For batched requests, either use a model that has a padding token, or specify your own # model_inputs = tokenizer(texts, return_tensors="pt", padding=True).to(model.device) model_inputs = tokenizer(texts, return_tensors="pt").to(model.device) # 3. Instantiate logits_processor per each generate, and call generate() xgr_logits_processor = xgr.contrib.hf.LogitsProcessor(compiled_grammar) generated_ids = model.generate( **model_inputs, max_new_tokens=512, logits_processor=[xgr_logits_processor] ) # 4. Post-process outputs and print out response generated_ids = [ output_ids[len(input_ids) :] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids) ] responses = tokenizer.batch_decode(generated_ids, skip_special_tokens=True) for response in responses: print(response, end="\n\n") xgrammar-0.2.3/include/000077500000000000000000000000001521764210300147565ustar00rootroot00000000000000xgrammar-0.2.3/include/module.modulemap000066400000000000000000000001131521764210300201430ustar00rootroot00000000000000module XGrammar { umbrella header "xgrammar/xgrammar.h" export * } xgrammar-0.2.3/include/xgrammar/000077500000000000000000000000001521764210300165745ustar00rootroot00000000000000xgrammar-0.2.3/include/xgrammar/compiler.h000066400000000000000000000074071521764210300205670ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/compiler.h * \brief The header for the compiler. */ #ifndef XGRAMMAR_COMPILER_H_ #define XGRAMMAR_COMPILER_H_ #include #include #include #include #include #include #include #include #include "xgrammar/exception.h" namespace xgrammar { /*! * \brief The compiled grammar of a GrammarMatcher. It contains the preprocessing results of the * grammar and tokenizer. */ class CompiledGrammar { public: /*! \brief Get the associated grammar. */ Grammar GetGrammar() const; /*! \brief Get the associated tokenizer info. */ TokenizerInfo GetTokenizerInfo() const; /*! \brief Return the approximate memory usage of the grammar in bytes. */ std::size_t MemorySizeBytes() const; /*! \brief Return the serialized JSON string of the compiled grammar. */ std::string SerializeJSON() const; /*! \brief Deserialize a compiled grammar from a JSON string and tokenizer info. */ static std::variant DeserializeJSON( const std::string& json_string, const TokenizerInfo& tokenizer_info ); XGRAMMAR_DEFINE_PIMPL_METHODS(CompiledGrammar); }; /*! * \brief A cache to get the compiled grammar for grammar or schema. This class avoids * redundant preprocessing of the grammar or schema when constructing a CompiledGrammar. * \note This class is associated with a vocabulary when constructed. The vocabulary is used to * create every compiled grammar. If multiple toke tables are used to create init * contexts, an instance of this class for each vocabulary should be created. */ class GrammarCompiler { public: /*! * \brief Construct a GrammarCompiler with a vocabulary. This class will always * create compiled grammars with this vocabulary. * \param tokenizer_info The tokenizer info. * \param max_threads The maximum number of threads to use for compiling grammars. * \param cache_enabled Whether to enable the cache. * \param max_memory_bytes The maximum memory usage in bytes. */ GrammarCompiler( const TokenizerInfo& tokenizer_info, int max_threads = 8, bool cache_enabled = true, int64_t max_memory_bytes = -1 // unlimited ); /*! \brief Get the compiled grammar for a JSON schema string. */ CompiledGrammar CompileJSONSchema( const std::string& schema, bool any_whitespace = true, std::optional indent = std::nullopt, std::optional> separators = std::nullopt, bool strict_mode = true, std::optional max_whitespace_cnt = std::nullopt, bool any_order = false ); /*! \brief Get the compiled grammar for pure JSON. */ CompiledGrammar CompileBuiltinJSONGrammar(); /*! \brief Get the compiled grammar for a grammar. */ CompiledGrammar CompileGrammar(const Grammar& grammar); /*! \brief Get the compiled grammar for a grammar. */ CompiledGrammar CompileGrammar( const std::string& ebnf_str, const std::string& root_rule_name = "root" ); /*! \brief Get the compiled grammar for a structural tag. */ CompiledGrammar CompileStructuralTag(const std::string& structural_tag_json); /*! \brief Get the compiled grammar for a regex. */ CompiledGrammar CompileRegex(const std::string& regex); /*! \brief Clear the internal cache of compiled grammars. */ void ClearCache(); /*! \brief Return the approximate memory usage of the compiler in bytes. */ int64_t GetCacheSizeBytes() const; /*! \brief Return the approximate memory usage of the compiler in bytes. -1 means unlimited. */ int64_t CacheLimitBytes() const; XGRAMMAR_DEFINE_PIMPL_METHODS(GrammarCompiler); }; } // namespace xgrammar #endif // XGRAMMAR_COMPILER_H_ xgrammar-0.2.3/include/xgrammar/config.h000066400000000000000000000014461521764210300202170ustar00rootroot00000000000000/*! * Copyright (c) 2025 by Contributors * \file xgrammar/config.h * \brief Global configuration for XGrammar. */ #ifndef XGRAMMAR_CONFIG_H_ #define XGRAMMAR_CONFIG_H_ #include namespace xgrammar { /*! * \brief Set the maximum recursion depth for the grammar. * \param max_recursion_depth The maximum recursion depth. */ void SetMaxRecursionDepth(int max_recursion_depth); /*! * \brief Get the maximum recursion depth for the grammar. * \return The maximum recursion depth. */ int GetMaxRecursionDepth(); /*! * \brief Get the serialization version for the grammar. * \return The serialization version. * \note This is used to check the compatibility of the serialized grammar. */ std::string GetSerializationVersion(); } // namespace xgrammar #endif // XGRAMMAR_CONFIG_H_ xgrammar-0.2.3/include/xgrammar/exception.h000066400000000000000000000050271521764210300207470ustar00rootroot00000000000000#ifndef XGRAMMAR_EXCEPTION_H #define XGRAMMAR_EXCEPTION_H #include #include #include namespace xgrammar { /************** Exception Definitions **************/ /*! * \brief Exception thrown when the version in the serialized data does not follow the current * serialization version. */ struct XGrammarError : std::runtime_error { XGrammarError(const std::string& message) : std::runtime_error(message) {} virtual std::string GetType() const { return "XGrammarError"; } }; struct DeserializeVersionError : XGrammarError { DeserializeVersionError(const std::string& message) : XGrammarError(std::string("Deserialize version error: ") + message) {} std::string GetType() const override { return "DeserializeVersionError"; } }; /*! * \brief Exception thrown when the JSON is invalid. */ struct InvalidJSONError : XGrammarError { InvalidJSONError(const std::string& message) : XGrammarError(std::string("Invalid JSON error: ") + message) {} std::string GetType() const override { return "InvalidJSONError"; } }; /*! * \brief Exception thrown when the serialized data does not follow the expected format. */ struct DeserializeFormatError : XGrammarError { DeserializeFormatError(const std::string& message) : XGrammarError(std::string("Deserialize format error: ") + message) {} std::string GetType() const override { return "DeserializeFormatError"; } }; /*! * \brief Exception thrown when the JSON schema is invalid or not satisfiable. */ struct InvalidJSONSchemaError : XGrammarError { InvalidJSONSchemaError(const std::string& message) : XGrammarError(std::string("Invalid JSON schema error: ") + message) {} std::string GetType() const override { return "InvalidJSONSchemaError"; } }; /*! * \brief Exception thrown when the structural tag is invalid. */ struct InvalidStructuralTagError : XGrammarError { InvalidStructuralTagError(const std::string& message) : XGrammarError(std::string("Invalid structural tag error: ") + message) {} std::string GetType() const override { return "InvalidStructuralTagError"; } }; /************** Union Exceptions **************/ /*! * \brief Represents a serialization error. */ using SerializationError = std::variant; /*! * \brief Represents an error from the structural tag conversion. */ using StructuralTagError = std::variant; } // namespace xgrammar #endif // XGRAMMAR_EXCEPTION_H xgrammar-0.2.3/include/xgrammar/grammar.h000066400000000000000000000171041521764210300203760ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar.h * \brief The header for the definition and construction of BNF grammar. */ #ifndef XGRAMMAR_GRAMMAR_H_ #define XGRAMMAR_GRAMMAR_H_ #include #include #include #include #include #include #include #include "xgrammar/exception.h" namespace xgrammar { struct StructuralTagItem { std::string begin; std::string schema; std::string end; bool operator==(const StructuralTagItem& other) const { return begin == other.begin && schema == other.schema && end == other.end; } }; /*! * \brief This class stores the abstract syntax tree (AST) of the Backus-Naur Form (BNF) grammar. * The BNF definition here is standard BNF, and the characters are represented using regex-style * character classes (e.g. [a-z], [^a-z]). * * \details * ### Rules * The BNF grammar AST consists of a set of rules. Each rule contains a name and a definition, and * corresponds to a production in the grammar. The definition of a rule is a GrammarExpr. Each rule * has a rule_id for reference. * * ### GrammarExprs * GrammarExpr is the definition of a rule or part of the definition of a rule. It can contain * elements, empty string, reference to other GrammarExprs, or reference to other rules. Each * GrammarExpr corresponds to a grammar_expr_id for reference. * * For example, in the following rule: rule ::= ("a" "b") | "c" * ("a" "b"), "c", ("a" "b") | "c" are all GrammarExprs. * * #### Types of GrammarExprs * Every GrammarExpr is represented by a type as well as a variable-length array containing its * data. GrammarExpr has several types: * - Byte string: a string of bytes (0~255). Supports UTF-8 strings. * - Character class: a range of characters (each character is a unicode codepoint), e.g. [a-z], * [ac-z]. Can be negated: [^a-z], [^ac-z]. Now only ascii chars is allowed in [], but this * expression can accept/reject unicode chars. * - Character class star: a star quantifier of a character class. e.g. [a-z]*, [^a-z]*. * - EmptyStr: an empty string, i.e. "" * - Rule reference: a reference to another rule * - Sequence: a sequence of grammar_exprs, e.g. ("a" "b"). These grammar_exprs are concatenated * together. * - Choices: a choice of grammar_exprs, e.g. ("a" "b") | "c". Each grammar_expr can be matched. * * #### Storage of GrammarExprs * Each type of GrammarExpr has a different data format. For the format of each type of GrammarExpr, * see docs in Grammar::Impl::GrammarExprType. * * We store all GrammarExprs in csr_matrix style. That is, they are stored consecutively in one * vector (data vector) and the starting position of each GrammarExpr is recorded in the indptr * vector. * * \remark The character class star GrammarExpr is for the special support for elements like [a-z]* * in the grammar. We add it to make the matching more efficient, as we can avoid recursion into * rules when matching a sequence of characters. It should be used like: * rule1 ::= ((element1 element2 rule2 ...) | ...) * rule2 ::= character_class_star_grammar_expr(id_of_a_character_class_grammar_expr) */ class Grammar { public: /*! * \brief Get the EBNF string of the grammar. */ std::string ToString() const; /*! * \brief Construct a BNF grammar with a EBNF-formatted string. The grammar will be normalized * (simplified) by default. * \param ebnf_string The EBNF-formatted string. * \param root_rule_name The name of the root rule. */ static Grammar FromEBNF( const std::string& ebnf_string, const std::string& root_rule_name = "root" ); /*! * \brief Construct a BNF grammar from the json schema string. The schema string should be in the * format of the schema of a JSON file. We will parse the schema and generate a BNF grammar. * \param schema The schema string. * \param indent The number of spaces for indentation. If set to std::nullopt, the output will be * in one line. Default: 2. * \param separators Two separators used in the schema: comma and colon. Examples: {",", ":"}, * {", ", ": "}. If std::nullopt, the default separators will be used: {",", ": "} when the * indent is not nullopt, and {", ", ": "} otherwise. This follows the convention in python * json.dumps(). Default: std::nullopt. * \param strict_mode Whether to use strict mode. In strict mode, the generated grammar will not * allow properties and items that is not specified in the schema. This is equivalent to * setting unevaluatedProperties and unevaluatedItems to false. * * This helps LLM to generate accurate output in the grammar-guided generation with JSON * schema. Default: true. */ static Grammar FromJSONSchema( const std::string& schema, bool any_whitespace = true, std::optional indent = std::nullopt, std::optional> separators = std::nullopt, bool strict_mode = true, std::optional max_whitespace_cnt = std::nullopt, bool print_converted_ebnf = false, bool any_order = false ); /*! * \brief Construct a grammar from a regular expression string. * \param regex The regular expression string. * \param print_converted_ebnf This method will convert the regex to EBNF first. If this is true, * the converted EBNF string will be printed. For debugging purpose. Default: false. */ static Grammar FromRegex(const std::string& regex, bool print_converted_ebnf = false); /*! * \brief Construct a grammar from a structural tag string. * \param structural_tag_json The structural tag string. * \param tokenizer_info Optional tokenizer info for resolving string token references. */ static std::variant FromStructuralTag( const std::string& structural_tag_json, const std::optional& tokenizer_info = std::nullopt ); /*! * \brief Get the grammar of standard JSON format. We have built-in support for JSON. * \return The grammar of standard JSON format. */ static Grammar BuiltinJSONGrammar(); /*! * \brief Create a grammar that matches any of the grammars in the list. That is equivalent to * using the `|` operator to concatenate the grammars in the list. * \param grammars The grammars to create the union of. * \returns The union of the grammars. */ static Grammar Union(const std::vector& grammars); /*! * \brief Create a grammar that matches the concatenation of the grammars in the list. That is * equivalent to using the `+` operator to concatenate the grammars in the list. * \param grammars The grammars to create the concatenation of. * \returns The concatenation of the grammars. */ static Grammar Concat(const std::vector& grammars); /*! * \brief Print a BNF grammar. * \param os The output stream. * \param grammar The grammar to print. * \return The output stream. */ friend std::ostream& operator<<(std::ostream& os, const Grammar& grammar); /*! * \brief Return the serialized JSON string of the grammar. * \return The serialized JSON string. */ std::string SerializeJSON() const; /*! * \brief Deserialize a grammar from a JSON string. * \param json_string The JSON string to deserialize. * \return If the deserialization is successful, return the grammar. Otherwise, return a runtime * error with the error message. */ static std::variant DeserializeJSON(const std::string& json_string); XGRAMMAR_DEFINE_PIMPL_METHODS(Grammar); }; } // namespace xgrammar #endif // XGRAMMAR_GRAMMAR_H_ xgrammar-0.2.3/include/xgrammar/matcher.h000066400000000000000000000232571521764210300204010ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/matcher.h * \brief The header for the matcher. */ #ifndef XGRAMMAR_MATCHER_H_ #define XGRAMMAR_MATCHER_H_ #include #include #include #include #include #include #include #include namespace xgrammar { int32_t GetBitmaskSize(int vocab_size); DLDataType GetBitmaskDLType(); void _DebugGetMaskedTokensFromBitmask( std::vector* rejected_tokens, const DLTensor& token_bitmask, int vocab_size, int index = 0 ); std::pair _IsSingleTokenBitmask(const DLTensor& bitmask, int vocab_size, int index); void ApplyTokenBitmaskInplaceCPU( DLTensor* logits, const DLTensor& bitmask, int vocab_size = -1, std::optional> indices = std::nullopt ); /*! * \brief A stateful matcher to match tokens to the specified BNF grammar. This class is the core * logic of the grammar-guided generation. * * \details This class implements the non-deterministic pushdown automaton (NPDA) matching algorithm * to match characters to a BNF grammar. It keep track of the current state of the matching process * by maintaining several stacks internally as possible paths in the NPDA. It also supports * backtracking. * * It is particularly capable of finding the set of tokens that are acceptable for the next step * and storing them in a bitmask. This aids in grammar-guided generation. * * \example * \code * Tokenizer tokenizer = ...; * auto compiled_grammar = GrammarMatcher::CreateCompiledGrammar(grammar, * tokenizer->PostProcessedVocab()); * GrammarMatcher matcher(compiled_grammar, 10); * matcher->AcceptToken(67); * * // Construct a DLTensor with shape (tokenizer.GetVocabSize() + 31) / 32, and dtype int32. * DLTensor next_token_bitmask = ...; * matcher->FillNextTokenBitmask(&next_token_bitmask); * * // Rollback is supported * matcher->Rollback(1); * \endcode */ class GrammarMatcher { public: /*! * \brief Construct a GrammarMatcher from the preprocessing result of type * CompiledGrammar. * \param compiled_grammar The compiled grammar. It is obtained through * CreateCompiledGrammar as a result of preprocessing the grammar and tokenizer. */ GrammarMatcher( const CompiledGrammar& compiled_grammar, std::optional> override_stop_tokens = std::nullopt, bool terminate_without_stop_token = false, int max_rollback_tokens = -1 ); /*! * \brief Accept one token and update the state of the matcher. * \param token_id The id of the token to accept. * \return Whether the token is accepted. * \note Termination state. * When the end of the root rule is reached, the matcher can only accept the stop token. * The matcher is terminated after accepting the stop token, i.e. no AcceptToken or * FindNextTokenMask operations can be performed. The termination state can be canceled * using Rollback(). */ bool AcceptToken(int32_t token_id, bool debug_print = false); /*! * \brief Accept a string and update the state of the matcher. The whole string is considered * as one step in rollback. It is used to complement the functionality of AcceptToken, and * AcceptToken should always be used to accept tokens. * \param input_str The string to be accepted. * \param debug_print Whether to print information about the internal state of the matcher. * \return Whether the string is accepted. */ bool AcceptString(const std::string& input_str, bool debug_print = false); /*! * \brief Get the set of tokens that are acceptable for the next step and store them in a * bitmask. * \param next_token_bitmask The bitmask to store the result. The bitmask must be pre-allocated * and with shape (GetBitmaskSize(),) and dtype int32. * \return Whether the bitmask need to be applied (not all-true). */ bool FillNextTokenBitmask(DLTensor* next_token_bitmask, int index = 0, bool debug_print = false); /*! * \brief Traverse a draft token tree and fill the token bitmask for each node. * * This function performs a DFS traversal of the speculative decoding tree and fills * the token bitmask for each node based on grammar constraints. * * \param retrieve_next_token DLTensor where retrieve_next_token[i] gives the index of * the child node of node i, or -1 if no child exists. * \param retrieve_next_sibling DLTensor where retrieve_next_sibling[i] gives the index of * the sibling node of node i, or -1 if no sibling exists. * \param draft_tokens DLTensor of draft token ids at each node. * \param token_bitmask DLTensor to store the bitmask (2D: num_nodes x bitmask_size). * \param time_threshold Maximum allowed time in seconds for the DFS traversal. * If the traversal exceeds this threshold, it returns false. * A value <= 0 disables the timeout (default: -1.0). * \return true if the traversal completed successfully, false if it timed out. */ bool TraverseDraftTree( const DLTensor* retrieve_next_token, const DLTensor* retrieve_next_sibling, const DLTensor* draft_tokens, DLTensor* token_bitmask, double time_threshold = -1.0 ); /*! * \brief Find the jump-forward string for jump-forward decoding. This is the longest string that will be valid according to the current syntax. * \note This method does not change the grammar state. */ std::string FindJumpForwardString(); /*! * \brief Rollback the matcher to a previous state. * \param num_tokens The number of tokens to rollback. It cannot exceed the current number of * steps, nor can it exceed the specified maximum number of rollback tokens. */ void Rollback(int num_tokens = 1); /*! * \brief Check if the matcher has accepted the stop token and terminated. * \sa AcceptToken */ bool IsTerminated() const; /*! * \brief Check if the grammar's root rule has been fully matched by the input accepted so far. * Unlike IsTerminated(), this does not require the stop token to have been accepted. * \sa IsTerminated, AcceptToken */ bool IsCompleted() const; /*! \brief Reset the matcher to the initial state. */ void Reset(); /*! * \brief Fork the matcher. Returns a new GrammarMatcher with a deep copy of all state except * compiled_grammar and tokenizer_info, which are shared with this matcher. */ GrammarMatcher Fork() const; /*! \brief Get the maximum number of rollback tokens allowed. */ int GetMaxRollbackTokens() const; const std::vector& GetStopTokenIds() const; /*! \brief Print the internal state of the matcher. This is only used for debugging. The * representation of the internal state is subject to change. */ std::string _DebugPrintInternalState() const; XGRAMMAR_DEFINE_PIMPL_METHODS(GrammarMatcher); }; /*! * \brief A batched version of GrammarMatcher for better efficiency. It supports batch processing * of multiple GrammarMatcher objects in parallel. * * \details This class provides batched versions of the core methods of GrammarMatcher, including * FillNextTokenBitmask, AcceptString, and AcceptToken. It utilizes multi-threading to process * multiple GrammarMatcher objects simultaneously, significantly improving efficiency when dealing * with a large number of matchers. */ class BatchGrammarMatcher { public: BatchGrammarMatcher(std::variant max_threads = "auto"); /*! \brief A batched version of FillNextTokenBitmask for better efficiency. \param matchers The array of GrammarMatcher objects. \param next_token_bitmask The pre-allocated DLTensor to store the result bitmasks. \param indices The optional array of indices to specify which matcher corresponds to which slice of the bitmask tensor. If not provided, all matchers will write to the corresponding indices(matchers[i] to next_token_bitmask[i]). \param debug_print Whether to print debug information. Default is false. */ void BatchFillNextTokenBitmask( std::vector* matchers, DLTensor* next_token_bitmask, const std::optional>& indices = std::nullopt, bool debug_print = false ); /*! * \brief A batched version of AcceptString for better efficiency. * \param matchers The array of GrammarMatcher objects. * \param input_strs The array of input strings to be accepted. * \param debug_print Whether to print debug information. Default is false. * \return A vector of bytes indicating whether each string is accepted. */ static std::vector BatchAcceptString( std::vector* matchers, const std::vector& input_strs, bool debug_print = false ); /*! * \brief A batched version of AcceptToken for better efficiency. * \param matchers The array of GrammarMatcher objects. * \param token_ids The array of token ids to be accepted. * \param debug_print Whether to print debug information. Default is false. * \return A vector of bytes indicating whether each token is accepted. */ static std::vector BatchAcceptToken( std::vector* matchers, const std::vector& token_ids, bool debug_print = false ); /*! * \brief A batched version of Rollback for better efficiency. * \param matchers The array of GrammarMatcher objects. * \param num_tokens The array of the number of tokens to rollback for each matcher. */ static void BatchRollback( std::vector* matchers, const std::vector& num_tokens ); XGRAMMAR_DEFINE_PIMPL_METHODS(BatchGrammarMatcher); }; } // namespace xgrammar #endif // XGRAMMAR_MATCHER_H_ xgrammar-0.2.3/include/xgrammar/object.h000066400000000000000000000046751521764210300202270ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/object.h * \brief Utilities for creating objects. */ #ifndef XGRAMMAR_OBJECT_H_ #define XGRAMMAR_OBJECT_H_ #include // IWYU pragma: keep #include // IWYU pragma: keep namespace xgrammar { /*! * \brief A tag type for creating a null object. */ struct NullObj {}; /*! * \brief This macro defines the methods for the PImpl classes. * \details Many classes in xgrammar are PImpl classes. PImpl classes only stores a shared pointer * to the implementation. This allows reference-counter-based memory management and efficient * object copy and passing. We always expose PImpl classes to Python to control over object sharing * and memory management. Note simple and critical classes should not be defined as PImpl classes, * but as normal classes for better efficiency. */ #define XGRAMMAR_DEFINE_PIMPL_METHODS(TypeName) \ public: \ class Impl; \ /* Construct a null object. Note operating on a null object will fail. */ \ explicit TypeName(NullObj) : pimpl_(nullptr) {} \ /* Construct object with a shared pointer to impl. */ \ explicit TypeName(std::shared_ptr pimpl) : pimpl_(std::move(pimpl)) {} \ TypeName(const TypeName& other) = default; \ TypeName(TypeName&& other) noexcept = default; \ TypeName& operator=(const TypeName& other) = default; \ TypeName& operator=(TypeName&& other) noexcept = default; \ bool IsNull() const { return pimpl_ == nullptr; } \ /* Access the impl pointer. Useful in implementation. */ \ Impl* ImplPtr() { return pimpl_.get(); } \ const Impl* ImplPtr() const { return pimpl_.get(); } \ Impl* operator->() { return pimpl_.get(); } \ const Impl* operator->() const { return pimpl_.get(); } \ \ private: \ std::shared_ptr pimpl_ } // namespace xgrammar #endif // XGRAMMAR_OBJECT_H_ xgrammar-0.2.3/include/xgrammar/tokenizer_info.h000066400000000000000000000046361521764210300220030ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/tokenizer_info.h * \brief The header for the tokenizer info. */ #ifndef XGRAMMAR_TOKENIZER_INFO_H_ #define XGRAMMAR_TOKENIZER_INFO_H_ #include #include #include #include #include #include #include "xgrammar/exception.h" namespace xgrammar { enum class VocabType : int { RAW = 0, BYTE_FALLBACK = 1, BYTE_LEVEL = 2, }; class TokenizerInfo { public: TokenizerInfo( const std::vector& encoded_vocab, VocabType vocab_type = VocabType::RAW, std::optional vocab_size = std::nullopt, std::optional> stop_token_ids = std::nullopt, bool add_prefix_space = false ); VocabType GetVocabType() const; bool GetAddPrefixSpace() const; int GetVocabSize() const; const std::vector& GetDecodedVocab() const; const std::vector& GetStopTokenIds() const; const std::vector& GetSpecialTokenIds() const; const std::vector>& GetSortedDecodedVocab() const; const std::vector& GetTrieSubtreeNodesRange() const; std::string DumpMetadata() const; /*! * \brief Create a tokenizer info from a vocabulary and metadata. * \param encoded_vocab The encoded vocabulary. * \param metadata The metadata. * \return The tokenizer info. */ static TokenizerInfo FromVocabAndMetadata( const std::vector& encoded_vocab, const std::string& metadata ); /*! * \brief Detect the metadata from a Hugging Face backend string. * \param backend_str The Hugging Face backend string. * \return The metadata. */ static std::string DetectMetadataFromHF(const std::string& backend_str); /*! * \brief Return the serialized JSON string of the tokenizer info. * \return The serialized JSON string. */ std::string SerializeJSON() const; /*! * \brief Deserialize a tokenizer info from a JSON string. * \param json_string The JSON string to deserialize. * \return If the deserialization is successful, return the tokenizer info. Otherwise, return a * runtime error with the error message. */ static std::variant DeserializeJSON( const std::string& json_string ); XGRAMMAR_DEFINE_PIMPL_METHODS(TokenizerInfo); }; } // namespace xgrammar #endif // XGRAMMAR_TOKENIZER_INFO_H_ xgrammar-0.2.3/include/xgrammar/xgrammar.h000066400000000000000000000006511521764210300205650ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/xgrammar.h * \brief The header for the support of grammar-guided generation. */ #ifndef XGRAMMAR_XGRAMMAR_H_ #define XGRAMMAR_XGRAMMAR_H_ #include #include #include #include #include #include #endif // XGRAMMAR_XGRAMMAR_H_ xgrammar-0.2.3/pyproject.toml000066400000000000000000000112251521764210300162500ustar00rootroot00000000000000[project] name = "xgrammar" version = "0.2.3" description = "Efficient, Flexible and Portable Structured Generation" authors = [{ name = "MLC Team" }] readme = "README.md" license = { text = "Apache 2.0" } classifiers = [ "License :: OSI Approved :: Apache Software License", "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Intended Audience :: Education", "Intended Audience :: Science/Research", ] keywords = ["machine learning", "inference"] requires-python = ">=3.8, <4" dependencies = [ "apache-tvm-ffi>=0.1.9", "pydantic", "torch>=1.10.0", "transformers>=4.38.0", "triton; platform_system == 'Linux' and platform_machine == 'x86_64'", "numpy", "typing-extensions>=4.9.0", ] [project.urls] Homepage = "https://xgrammar.mlc.ai/" GitHub = "https://github.com/mlc-ai/xgrammar" [project.optional-dependencies] test = [ "huggingface-hub[cli]", "protobuf", "pytest", "sentencepiece", "tiktoken", # transformers==4.50.0 has error on MacOS. # https://github.com/huggingface/transformers/issues/36906 "transformers<4.50.0; platform_system == 'Darwin'", ] metal = ["mlx-lm; platform_system == 'Darwin' and platform_machine == 'arm64'"] [build-system] requires = ["scikit-build-core>=0.10.0", "apache-tvm-ffi>=0.1.9"] build-backend = "scikit_build_core.build" [tool.scikit-build] minimum-version = "build-system.requires" # Build configuration build-dir = "build" build.verbose = true # CMake configuration cmake.version = "CMakeLists.txt" cmake.args = [] cmake.build-type = "RelWithDebInfo" # Logging logging.level = "INFO" # Wheel configuration wheel.packages = ["python/xgrammar"] wheel.install-dir = "xgrammar" # Source distribution configuration sdist.include = [ # Build files "/CMakeLists.txt", "/pyproject.toml", "/cmake/**/*", "/cpp/**/CMakeLists.txt", # Source code "/cpp/**/*.cc", "/cpp/**/*.cpp", "/cpp/**/*.h", "/include/**/*", "/python/xgrammar/**/*.py", "/python/xgrammar/py.typed", # Third party files "/3rdparty/**/*", # Documentation and metadata "/docs/**/*", "/LICENSE", "/README.md", "/NOTICE", # Tests "/tests/**/*", ] sdist.exclude = ["**/.git", "**/.github", "**/__pycache__", "**/*.pyc", "build", "dist"] # Editable install settings editable.rebuild = true editable.verbose = true [tool.pytest.ini_options] testpaths = ["tests"] addopts = "-rA --durations=0 --ignore=3rdparty" markers = ["hf_token_required: mark test as requiring a huggingface token"] [tool.mypy] strict = true [tool.ruff] include = ["python/**/*.py", "tests/**/*.py"] [tool.ruff.lint] # Never enforce `E501` (line length violations). ignore = ["C901", "E501", "E741", "F402", "F823", "E731"] select = ["C", "E", "F", "W"] [tool.ruff.lint.per-file-ignores] "__init__.py" = ["F401"] "tests/*" = ["E741"] [tool.ruff.lint.pylint] max-args = 10 [tool.black] exclude = "3rdparty/*" line-length = 100 skip-magic-trailing-comma = true [tool.isort] profile = "black" src_paths = ["python", "tests"] extend_skip = ["3rdparty"] line_length = 100 skip_gitignore = true [tool.cibuildwheel] build-verbosity = 1 # pypy doesn't play nice with pybind11 so skip pp* builds # pytorch stopped supporting Mac x64 back in 2.2 so there will be no Mac x64 wheels for python 3.13 so skip cp313-macosx_x86_64 # python 3.13 support is still early and wheels are missing for Linux aarch64 for pytorch so temporarily skip cp313-manylinux_aarch64 skip = [ "cp36-*", "cp37-*", "cp38-*", "*musllinux*", "cp313-macosx_x86_64", ] # pypy doesn't play nice with pybind11 build-frontend = "build[uv]" test-command = "pytest {project}/tests -m \"not hf_token_required\"" test-extras = ["test"] [tool.cibuildwheel.linux] archs = ["x86_64", "aarch64"] # Exclude libtvm_ffi.so: provided by apache-tvm-ffi at import time (see TVM-FFI Python Packaging docs) repair-wheel-command = "auditwheel repair --exclude libtvm_ffi.so -w {dest_dir} {wheel}" [tool.cibuildwheel.macos] archs = ["x86_64", "arm64"] environment = { MACOSX_DEPLOYMENT_TARGET = "10.14" } # Exclude libtvm_ffi.dylib: it is provided by apache-tvm-ffi at import time (see TVM-FFI Python Packaging docs). # --ignore-missing-dependencies is needed because delocate fails when resolving @rpath/libtvm_ffi.dylib # before the copy step; exclude only filters which found libs to copy. repair-wheel-command = "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v --exclude libtvm_ffi.dylib --ignore-missing-dependencies {wheel}" [tool.cibuildwheel.windows] archs = ["AMD64"] before-build = "pip install delvewheel" # Exclude tvm_ffi.dll: provided by apache-tvm-ffi at import time (see TVM-FFI Python Packaging docs) repair-wheel-command = "delvewheel repair --exclude tvm_ffi.dll -w {dest_dir} {wheel}" xgrammar-0.2.3/python/000077500000000000000000000000001521764210300146545ustar00rootroot00000000000000xgrammar-0.2.3/python/xgrammar/000077500000000000000000000000001521764210300164725ustar00rootroot00000000000000xgrammar-0.2.3/python/xgrammar/__init__.py000066400000000000000000000032511521764210300206040ustar00rootroot00000000000000from . import exception, load_binding, structural_tag, testing from .builtin_structural_tag import ( get_builtin_structural_tag, get_model_structural_tag, normalize_tool_choice, register_model_structural_tag, ) from .compiler import CompiledGrammar, GrammarCompiler from .config import ( get_max_recursion_depth, get_serialization_version, max_recursion_depth, set_max_recursion_depth, ) from .contrib import hf from .exception import ( DeserializeFormatError, DeserializeVersionError, InvalidJSONError, InvalidStructuralTagError, ) from .grammar import Grammar, StructuralTagItem from .matcher import ( BatchGrammarMatcher, GrammarMatcher, allocate_token_bitmask, apply_token_bitmask_inplace, bitmask_dtype, get_bitmask_shape, reset_token_bitmask, ) from .structural_tag import StructuralTag from .tokenizer_info import TokenizerInfo, VocabType __all__ = [ "exception", "structural_tag", "testing", "CompiledGrammar", "GrammarCompiler", "get_max_recursion_depth", "get_serialization_version", "max_recursion_depth", "set_max_recursion_depth", "hf", "DeserializeFormatError", "DeserializeVersionError", "InvalidJSONError", "InvalidStructuralTagError", "Grammar", "StructuralTagItem", "BatchGrammarMatcher", "GrammarMatcher", "allocate_token_bitmask", "apply_token_bitmask_inplace", "bitmask_dtype", "get_bitmask_shape", "reset_token_bitmask", "StructuralTag", "TokenizerInfo", "VocabType", "get_model_structural_tag", "normalize_tool_choice", "register_model_structural_tag", "get_builtin_structural_tag", ] xgrammar-0.2.3/python/xgrammar/base.py000066400000000000000000000063651521764210300177700ustar00rootroot00000000000000"""This module provides classes to handle C++ objects via tvm_ffi.""" import os from typing import Any, Union if os.environ.get("XGRAMMAR_BUILD_DOCS") != "1": from tvm_ffi import Object as _ffi_Object from .tvm_ffi_binding import _ffi_api as _core from .tvm_ffi_binding import config as _config_ffi from .tvm_ffi_binding.kernels import _ffi_api as _kernels_ffi from .tvm_ffi_binding.testing import _ffi_api as _testing_ffi from .tvm_ffi_binding.testing.grammar_functor import _ffi_api as _grammar_functor_ffi _core.testing = _testing_ffi _core.testing.grammar_functor = _grammar_functor_ffi _core.kernels = _kernels_ffi _core.config = _config_ffi else: _ffi_Object: Any = None # type: ignore[misc, assignment] _core: Any = None class XGRObject: """The base class for all objects in XGrammar. This class provides methods to handle the C++ object through a tvm_ffi Object (or its derived class) held by each instance. In subclasses, the FFI object should be initialized via _create_from_handle, or via _init_handle called within the __init__ method, and should not be modified afterwards. Subclasses should use the _handle property to access the underlying FFI object. When comparing two objects, equality is checked by comparing the underlying FFI objects. For performance considerations, objects in XGrammar should be lightweight and only maintain a handle to the C++ objects. Heavy operations should be performed on the C++ side. """ @classmethod def _create_from_handle(cls, handle: Union["_ffi_Object", Any]) -> "XGRObject": """Construct an object of the class from an FFI object (tvm_ffi Object or derived). Parameters ---------- cls The class of the object. handle The FFI object (e.g. from _core.Grammar, _core.CompiledGrammar, etc.). Returns ------- obj : XGRObject An object of type cls. """ obj = cls.__new__(cls) obj.__handle = handle return obj def _init_handle(self, handle: Union["_ffi_Object", Any]) -> None: """Initialize an object with an FFI handle. This method should be called in the __init__ method of the subclasses of XGRObject to initialize the underlying FFI object. Parameters ---------- handle The FFI object (e.g. from _core.GrammarCompiler, _core.GrammarMatcher, etc.). """ self.__handle = handle @property def _handle(self) -> Union["_ffi_Object", Any]: """Get the underlying FFI object (tvm_ffi Object or derived). Returns ------- handle The FFI object used for C++ communication. """ return self.__handle def __eq__(self, other: object) -> bool: """Compare two XGrammar objects by comparing their underlying FFI objects. Parameters ---------- other : object The other object to compare with. Returns ------- equal : bool Whether the two objects have the same underlying FFI object. """ if not isinstance(other, XGRObject): return NotImplemented return self._handle == other._handle xgrammar-0.2.3/python/xgrammar/builtin_structural_tag.py000066400000000000000000002137631521764210300236510ustar00rootroot00000000000000from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union from pydantic import TypeAdapter from .openai_tool_call_schema import ( AllowedToolChoiceParam, BuiltinToolChoiceParam, BuiltinToolParam, FunctionDefinition, FunctionToolParam, NamedToolChoiceParam, ToolChoiceOptionParam, ToolParam, ) from .structural_tag import ( AnyTextFormat, ConstStringFormat, JSONSchemaFormat, RegexFormat, SequenceFormat, StructuralTag, TagFormat, TagsWithSeparatorFormat, TriggeredTagsFormat, ) # ---------- API Functions ---------- def get_model_structural_tag( model: str, tools: Optional[List[Union[ToolParam, dict]]] = None, tool_choice: Union[ToolChoiceOptionParam, dict, None] = "auto", reasoning: bool = True, force_reasoning: bool = False, any_order: bool = False, exclude_special_tokens: bool = True, ) -> StructuralTag: r"""Get a structural tag for a model's reasoning and tool-call output format. Use this function when a serving engine needs a structural tag that matches a model's tool-call syntax. Pass the model format, the available tools, and the desired tool choice policy. This API is designed to resemble OpenAI Chat Completions API. Function tools use the OpenAI Chat Completions shape: ``{"type": "function", "function": {...}}``. Builtin tools use a compact shape: - ``type`` is the provider-level builtin tool type, such as ``"web_search_preview"``. - ``name`` is the exact tool name that may appear in model output. If it is omitted, ``type`` is used as the output name. - ``parameters`` is the JSON schema used to constrain the arguments emitted by the model. Examples -------- Ordinary function tool: .. code-block:: python structural_tag = get_model_structural_tag( "llama", tools=[ { "type": "function", "function": { "name": "get_weather", "parameters": { "type": "object", "properties": {"location": {"type": "string"}}, }, }, } ], ) Harmony with a builtin web search tool: .. code-block:: python structural_tag = get_model_structural_tag( "harmony", tools=[ { "type": "web_search_preview", "name": "browser.search", "parameters": { "type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"], }, } ], ) Force an ordinary function tool: .. code-block:: python structural_tag = get_model_structural_tag( "llama", tools=[ { "type": "function", "function": { "name": "get_weather", "parameters": { "type": "object", "properties": {"location": {"type": "string"}}, }, }, } ], tool_choice={ "type": "function", "function": {"name": "get_weather"}, }, ) Force a builtin tool by type and output name: .. code-block:: python structural_tag = get_model_structural_tag( "harmony", tools=[...], tool_choice={"type": "web_search_preview"}, ) Allow only a subset of tools: .. code-block:: python structural_tag = get_model_structural_tag( "harmony", tools=[...], tool_choice={ "type": "allowed_tools", "allowed_tools": { "mode": "auto", "tools": [ {"type": "function", "function": {"name": "get_weather"}}, {"type": "web_search_preview"}, ], }, }, ) Parameters ---------- model : str The model type of the structural tag template. It should be one of the registered values. tools : Optional[List[Union[ToolParam, dict]]] Function and builtin tools available to the model. Function tools use the Chat Completions shape. Builtin tools use ``type`` plus optional ``name`` and ``parameters`` fields. Defaults to ``None``, which is treated as an empty list. tool_choice : Union[ToolChoiceOptionParam, dict, None] Controls whether the model may or must call tools. Defaults to ``"auto"``. - ``"auto"`` lets the model choose between text output and tool calls. - ``None`` is treated the same as ``"auto"``. - ``"none"`` disables all tools. - ``"required"`` requires at least one available tool. - ``{"type": "function", "function": {"name": ...}}`` forces one function tool. - ``{"type": }`` forces one builtin tool. Builtin tool choices are matched by ``type``. - ``{"type": "allowed_tools", "allowed_tools": ...}`` limits the available tools before applying its ``mode``. Its ``tools`` list may contain both function refs and builtin refs. Builtin refs are matched by ``type``. reasoning : bool Whether to enable the reasoning part. Some models, such as Qwen 3.6 and DeepSeek V4, support both reasoning and non-reasoning modes. If ``False``, use the non-reasoning mode. For models that do not support reasoning, this has no effect. For models that only support reasoning, ``False`` means reasoning with empty content. force_reasoning : bool Deprecated. Control whether to keep the reasoning part but leave its content empty. Now we will embed the model's specific behavior into the structural tag function, so only controlling ``reasoning`` is enough. any_order : bool Relax object property ordering for every tool-argument schema. When ``True``, ``any_order=True`` is applied to every :class:`JSONSchemaFormat` in the generated structural tag, so each tool's arguments may be emitted in any property order (see :class:`JSONSchemaFormat` for the exact semantics). When ``False`` (default), the declared property order is kept with full validation. Default: ``False``. exclude_special_tokens : bool Whether to forbid model special tokens (such as ```` and ````) from appearing inside the free-text and triggered-text spans of the structural tag. Defaults to ``True``, which keeps the free-text spans constrained to exclude those tokens. Set to ``False`` to allow them to appear as plain text. For models that have no special tokens to exclude (such as ``"harmony"``), this has no effect. Notes ----- If a tool's ``parameters`` field is omitted or ``None``, its generated arguments are unconstrained JSON. If a function tool has ``strict=False``, its ``parameters`` schema is also treated as unconstrained. Returns ------- StructuralTag A structural tag for function calling format. Raises ------ ValueError If tool lists, tool choices, or required tool availability are invalid. """ func = _structural_tag_registry.get(model) if func is None: supported = list(_structural_tag_registry.keys()) raise ValueError(f"Unknown format type: {model}, supported types: {supported}") function_tools, builtin_tools, simplified_tool_choice = normalize_tool_choice( tools, tool_choice ) return func( function_tools, builtin_tools, simplified_tool_choice, reasoning, any_order=any_order, exclude_special_tokens=exclude_special_tokens, ) # ---------- Helper Functions And Constants ---------- SimplifiedToolChoice = Literal["auto", "required", "forced"] BuiltinStructuralTagFn = Callable[..., StructuralTag] _TOOL_ADAPTER = TypeAdapter(ToolParam) _TOOL_CHOICE_ADAPTER = TypeAdapter(ToolChoiceOptionParam) _structural_tag_registry: Dict[str, BuiltinStructuralTagFn] = {} def normalize_tool_choice( tools: Optional[List[Union[ToolParam, dict]]] = None, tool_choice: Union[ToolChoiceOptionParam, dict, None] = "auto", ) -> Tuple[List[FunctionToolParam], List[BuiltinToolParam], SimplifiedToolChoice]: r"""Normalize tools and tool choice for structural tag builders. This helper exposes the model-independent part of :func:`get_model_structural_tag`. It is intended for serving engines that want to own their model-specific structural tag templates while reusing OpenAI-style tool and tool-choice handling. The return value is not a new public tool-calling protocol. It is a compact prepared form for structural tag builder functions: - ordinary function tools are returned as ``FunctionToolParam`` objects; - builtin/server tools are returned as ``BuiltinToolParam`` objects; - public tool-choice values are simplified to ``"auto"``, ``"required"``, or ``"forced"``. Parameters ---------- tools : Optional[List[Union[ToolParam, dict]]] Function and builtin tools available to the model. Function tools use the OpenAI Chat Completions shape, ``{"type": "function", "function": {...}}``. Builtin tools use ``type`` plus optional ``name`` and ``parameters`` fields. ``None`` is treated as an empty list. tool_choice : Union[ToolChoiceOptionParam, dict, None] Controls whether the model may or must call tools. This accepts the same values as :func:`get_model_structural_tag`: - ``"auto"`` keeps all available tools and lets the builder allow text or tool calls. - ``None`` is treated as ``"auto"``. - ``"none"`` clears all tools and returns simplified choice ``"auto"``. Builders already interpret auto with no tools as text-only. - ``"required"`` keeps all available tools and requires at least one function or builtin tool to remain available. - ``{"type": "function", "function": {"name": ...}}`` filters to the named function tool and returns simplified choice ``"forced"``. - ``{"type": }`` filters to exactly one builtin tool whose ``type`` matches and returns simplified choice ``"forced"``. - ``{"type": "allowed_tools", "allowed_tools": ...}`` filters to the referenced tools and returns the nested allowed-tools ``mode`` as the simplified choice. Returns ------- Tuple[List[FunctionToolParam], List[BuiltinToolParam], SimplifiedToolChoice] A tuple of ``(function_tools, builtin_tools, simplified_tool_choice)`` ready to pass to a model-specific structural tag builder. Raises ------ ValueError If ``tools`` is not a list, a referenced tool is missing, a builtin tool choice does not match exactly one builtin tool, ``required`` leaves no available tools, or ``forced`` does not resolve to exactly one tool. Examples -------- Build tool-choice handling with an external model-specific builder: .. code-block:: python function_tools, builtin_tools, tool_choice = normalize_tool_choice( tools=[ { "type": "function", "function": { "name": "get_weather", "parameters": { "type": "object", "properties": {"location": {"type": "string"}}, }, }, } ], tool_choice={ "type": "function", "function": {"name": "get_weather"}, }, ) structural_tag = build_my_model_structural_tag( function_tools, builtin_tools, tool_choice, reasoning=True, ) """ if tools is None: tools = [] if not isinstance(tools, list): raise ValueError("The 'tools' argument must be a list.") normalized_tools = [ ( tool if isinstance(tool, (FunctionToolParam, BuiltinToolParam)) else _TOOL_ADAPTER.validate_python(tool) ) for tool in tools ] # Model-specific functions need separate lists because builtin tools may use # a different output channel or recipient from ordinary function tools. function_tools = [tool for tool in normalized_tools if isinstance(tool, FunctionToolParam)] builtin_tools = [tool for tool in normalized_tools if isinstance(tool, BuiltinToolParam)] if tool_choice is None: normalized_tool_choice: ToolChoiceOptionParam = "auto" else: normalized_tool_choice = _TOOL_CHOICE_ADAPTER.validate_python(tool_choice) simplified_tool_choice: SimplifiedToolChoice if isinstance(normalized_tool_choice, AllowedToolChoiceParam): function_tools, builtin_tools = _filter_allowed_tools( function_tools, builtin_tools, normalized_tool_choice ) simplified_tool_choice = normalized_tool_choice.allowed_tools.mode elif isinstance(normalized_tool_choice, NamedToolChoiceParam): tool_name = normalized_tool_choice.function.name function_tools = [tool for tool in function_tools if tool.function.name == tool_name] if not function_tools: raise ValueError(f"The tool with name '{tool_name}' is not found in the tools list.") builtin_tools = [] simplified_tool_choice = "forced" elif isinstance(normalized_tool_choice, BuiltinToolChoiceParam): function_tools = [] builtin_tools = [tool for tool in builtin_tools if tool.type == normalized_tool_choice.type] if len(builtin_tools) != 1: raise ValueError( "Builtin tool choice must match exactly one builtin tool, " f"got {len(builtin_tools)} matches." ) simplified_tool_choice = "forced" elif normalized_tool_choice == "none": # The internal functions already treat auto with no tools as text-only. function_tools = [] builtin_tools = [] simplified_tool_choice = "auto" else: simplified_tool_choice = normalized_tool_choice if simplified_tool_choice == "required" and not function_tools and not builtin_tools: raise ValueError( "The 'tools' list is empty, which is not allowed when " "'tool_choice' is 'required'." ) if simplified_tool_choice == "forced" and len(function_tools) + len(builtin_tools) != 1: raise ValueError("Forced tool choice must resolve to exactly one tool.") return function_tools, builtin_tools, simplified_tool_choice def _get_function_parameters( function: Union[FunctionDefinition, BuiltinToolParam] ) -> Union[Dict[str, Any], bool]: """Return the JSON schema used for constrained tool arguments. ``None`` parameters and non-strict function tools are intentionally mapped to ``True`` so the generated arguments remain syntactically constrained but schema-unconstrained. """ if isinstance(function, FunctionDefinition) and function.strict is False: return True if function.parameters is None: return True return function.parameters def _get_builtin_tool_name(tool: BuiltinToolParam) -> str: """Return the model-output name for a builtin tool.""" return tool.name or tool.type def _text_excludes(exclude_special_tokens: bool, tokens: List[str]) -> List[str]: """Resolve the tokens to forbid inside a structural tag's free-text spans. Built-in structural tags normally forbid model special tokens (such as ```` and ````) from appearing inside the free-text and triggered-text spans, so the model cannot emit them as plain text. Some downstream setups do not want this restriction. When ``exclude_special_tokens`` is ``True`` (the default), this returns *tokens* unchanged; when ``False``, it returns an empty list so nothing is excluded. """ return list(tokens) if exclude_special_tokens else [] def _filter_allowed_tools( tools: List[FunctionToolParam], builtin_tools: List[BuiltinToolParam], tool_choice: AllowedToolChoiceParam, ) -> Tuple[List[FunctionToolParam], List[BuiltinToolParam]]: """Filter tools according to a public allowed-tools tool choice.""" allowed_function_names = set() allowed_builtin_types = set() for allowed_tool in tool_choice.allowed_tools.tools: if allowed_tool.type == "function": if allowed_tool.function is None: raise ValueError("Allowed function tool references must include 'function'.") allowed_function_names.add(allowed_tool.function.name) else: allowed_builtin_types.add(allowed_tool.type) missing_function_names = allowed_function_names - {tool.function.name for tool in tools} if missing_function_names: raise ValueError( f"Allowed function tools are not found in the tools list: {missing_function_names}." ) filtered_builtin_tools = [tool for tool in builtin_tools if tool.type in allowed_builtin_types] matched_builtin_types = {tool.type for tool in filtered_builtin_tools} missing_builtin_refs = allowed_builtin_types - matched_builtin_types if missing_builtin_refs: raise ValueError( f"Allowed builtin tools are not found in the tools list: {missing_builtin_refs}." ) filtered_tools = [tool for tool in tools if tool.function.name in allowed_function_names] return filtered_tools, filtered_builtin_tools def register_model_structural_tag(name: str): """Register a model-specific structural tag function under *name*. The decorated function is stored in the internal registry so that :func:`get_model_structural_tag` can look it up by the ``model`` argument. Use this to add support for a new model format. Parameters ---------- name : str The model format key, e.g. ``"llama"``, ``"harmony"``. Examples -------- .. code-block:: python @register_model_structural_tag("my_model") def get_my_model_structural_tag( tools=None, builtin_tools=None, tool_choice="auto", reasoning=True, **kwargs, ): ... """ def decorator(func): _structural_tag_registry[name] = func return func return decorator # ---------- Each Built-in Structural Tag Function ---------- @register_model_structural_tag("llama") def get_llama_structural_tag( tools: Optional[List[FunctionToolParam]] = None, builtin_tools: Optional[List[BuiltinToolParam]] = None, tool_choice: Literal["auto", "required", "forced"] = "auto", reasoning: bool = True, any_order: bool = False, exclude_special_tokens: bool = True, **kwargs: Any, ) -> StructuralTag: """Get Llama style structural tag format. Corresponding model key: ``"llama"``. Reference: https://www.llama.com/docs/model-cards-and-prompt-formats/llama3_1/ Parameters are normalized by :func:`get_model_structural_tag` before this function is called: - ``tools``: a list of function tools. Each tool should have a ``function`` object containing ``name`` and ``parameters`` fields. - ``reasoning``: ignored because this format has no reasoning part. Supported models: - Meta-Llama-3 - Llama-3.1 - Llama-3.2 Returns ------- StructuralTag A structural tag for function calling format. This format is used by Llama 3 and other models that follow the same style. """ TOOL_NAME_PREFIX = '{"name": "' PARAMETERS_FIELD_PREFIX = '", "parameters": ' TOOL_OBJECT_BEGIN_PREFIX = '{"name": "' TOOL_OBJECT_PARAMETERS_PREFIX = '", "parameters": ' TOOLS_TRIGGER = '{"name": ' THINK_EXCLUDE_TOKENS = ["", ""] tools = tools or [] builtin_tools = builtin_tools or [] if tool_choice == "auto": tags = [] for tool in tools: function = tool.function parameters = _get_function_parameters(function) name = function.name tags.append( TagFormat( begin=(TOOL_OBJECT_BEGIN_PREFIX + name + TOOL_OBJECT_PARAMETERS_PREFIX), content=JSONSchemaFormat(json_schema=parameters, any_order=any_order), end="}", ) ) if len(tags) > 0: suffix_tag = TriggeredTagsFormat( triggers=[TOOLS_TRIGGER], tags=tags, excludes=_text_excludes(exclude_special_tokens, THINK_EXCLUDE_TOKENS), ) else: suffix_tag = AnyTextFormat( excludes=_text_excludes(exclude_special_tokens, THINK_EXCLUDE_TOKENS) ) elif tool_choice == "forced": if not tools: raise ValueError("Forced tool choice must resolve to exactly one tool.") function = tools[0].function suffix_tag = TagFormat( begin=(TOOL_NAME_PREFIX + function.name + PARAMETERS_FIELD_PREFIX), content=JSONSchemaFormat( json_schema=_get_function_parameters(function), any_order=any_order ), end="}", ) elif tool_choice == "required": tags = [] for tool in tools: function = tool.function parameters = _get_function_parameters(function) name = function.name tags.append( TagFormat( begin=(TOOL_OBJECT_BEGIN_PREFIX + name + TOOL_OBJECT_PARAMETERS_PREFIX), content=JSONSchemaFormat(json_schema=parameters, any_order=any_order), end="}", ) ) assert len(tags) > 0 suffix_tag = TagsWithSeparatorFormat(tags=tags, separator="", at_least_one=True) return StructuralTag(format=suffix_tag) @register_model_structural_tag("kimi") def get_kimi_structural_tag( tools: Optional[List[FunctionToolParam]] = None, builtin_tools: Optional[List[BuiltinToolParam]] = None, tool_choice: Literal["auto", "required", "forced"] = "auto", reasoning: bool = True, any_order: bool = False, exclude_special_tokens: bool = True, **kwargs: Any, ) -> StructuralTag: """Get Kimi-K2 style structural tag format. Corresponding model key: ``"kimi"``. Reference: https://huggingface.co/moonshotai/Kimi-K2-Instruct/blob/main/docs/tool_call_guidance.md Parameters are normalized by :func:`get_model_structural_tag` before this function is called: - ``tools``: a list of function tools. Each tool should have a ``function`` object containing ``name`` and ``parameters`` fields. - ``reasoning``: whether to enable reasoning mode. If ``False``, remove the reasoning part and constrain only the following part. Supported models: - Kimi-K2 - Kimi-K2.5 Returns ------- StructuralTag A structural tag template. This format is used by Kimi-K2 and other models that follow the same style. """ TOOL_CALL_BEGIN = "<|tool_call_begin|>" TOOL_CALL_BEGIN_PREFIX = f"{TOOL_CALL_BEGIN}functions." TOOL_CALL_SUFFIX = ":" TOOL_CALL_ARGUMENT_BEGIN = "<|tool_call_argument_begin|>" TOOL_CALL_END = "<|tool_call_end|>" TOOL_CALLS_SECTION_BEGIN = "<|tool_calls_section_begin|>" TOOL_CALLS_SECTION_END = "<|tool_calls_section_end|>" THINK_TAG_END = "" THINK_EXCLUDE_TOKENS = ["", ""] tools = tools or [] builtin_tools = builtin_tools or [] if tool_choice == "auto": tags = [] for tool in tools: function = tool.function parameters = _get_function_parameters(function) name = function.name tags.append( TagFormat( begin=f"{TOOL_CALL_BEGIN_PREFIX}{name}{TOOL_CALL_SUFFIX}", content=SequenceFormat( elements=[ RegexFormat(pattern=r"\d+"), ConstStringFormat(value=TOOL_CALL_ARGUMENT_BEGIN), JSONSchemaFormat(json_schema=parameters, any_order=any_order), ] ), end=TOOL_CALL_END, ) ) if len(tags) > 0: inner_tool_calls = TagsWithSeparatorFormat(tags=tags, separator="", at_least_one=True) tool_calls = TagFormat( begin=TOOL_CALLS_SECTION_BEGIN, content=inner_tool_calls, end=TOOL_CALLS_SECTION_END ) suffix_tag = TriggeredTagsFormat( triggers=[TOOL_CALLS_SECTION_BEGIN], tags=[tool_calls], excludes=_text_excludes( exclude_special_tokens, [*THINK_EXCLUDE_TOKENS, TOOL_CALL_BEGIN] ), ) else: suffix_tag = AnyTextFormat( excludes=_text_excludes(exclude_special_tokens, THINK_EXCLUDE_TOKENS) ) elif tool_choice == "forced": if not tools: raise ValueError("Forced tool choice must resolve to exactly one tool.") function = tools[0].function suffix_tag = SequenceFormat( elements=[ ConstStringFormat(value=TOOL_CALLS_SECTION_BEGIN), TagFormat( begin=f"{TOOL_CALL_BEGIN_PREFIX}{function.name}{TOOL_CALL_SUFFIX}", content=SequenceFormat( elements=[ RegexFormat(pattern=r"\d+"), ConstStringFormat(value=TOOL_CALL_ARGUMENT_BEGIN), JSONSchemaFormat( json_schema=_get_function_parameters(function), any_order=any_order ), ] ), end=TOOL_CALL_END, ), ConstStringFormat(value=TOOL_CALLS_SECTION_END), ] ) elif tool_choice == "required": tags = [] for tool in tools: function = tool.function parameters = _get_function_parameters(function) name = function.name tags.append( TagFormat( begin=f"{TOOL_CALL_BEGIN_PREFIX}{name}{TOOL_CALL_SUFFIX}", content=SequenceFormat( elements=[ RegexFormat(pattern=r"\d+"), ConstStringFormat(value=TOOL_CALL_ARGUMENT_BEGIN), JSONSchemaFormat(json_schema=parameters, any_order=any_order), ] ), end=TOOL_CALL_END, ) ) assert len(tags) > 0 suffix_tag = SequenceFormat( elements=[ ConstStringFormat(value=TOOL_CALLS_SECTION_BEGIN), TagsWithSeparatorFormat(tags=tags, separator="", at_least_one=True), ConstStringFormat(value=TOOL_CALLS_SECTION_END), ] ) if not reasoning: return StructuralTag(format=suffix_tag) prefix_tag = TagFormat(begin="", content=AnyTextFormat(), end=THINK_TAG_END) return StructuralTag(format=SequenceFormat(elements=[prefix_tag, suffix_tag])) @register_model_structural_tag("deepseek_r1") def get_deepseek_r1_structural_tag( tools: Optional[List[FunctionToolParam]] = None, builtin_tools: Optional[List[BuiltinToolParam]] = None, tool_choice: Literal["auto", "required", "forced"] = "auto", reasoning: bool = True, any_order: bool = False, exclude_special_tokens: bool = True, **kwargs: Any, ) -> StructuralTag: """Get DeepSeek-R1 style structural tag format. Corresponding model key: ``"deepseek_r1"``. Reference: https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/main/tokenizer_config.json Supported models: - DeepSeek-R1 - DeepSeek-R1-0528 """ TOOL_CALLS_BEGIN = "<｜tool▁calls▁begin｜>" TOOL_CALLS_END = "<｜tool▁calls▁end｜>" TOOL_CALL_BEGIN = "<｜tool▁call▁begin｜>" TOOL_CALL_END = "<｜tool▁call▁end｜>" TOOL_SEP = "<｜tool▁sep｜>" JSON_RENDER_BEGIN = "\n```json\n" JSON_RENDER_END = "\n```" THINK_TAG_END = "" THINK_EXCLUDE_TOKENS = ["", ""] tools = tools or [] builtin_tools = builtin_tools or [] if tool_choice == "auto": tags = [] for tool in tools: function = tool.function parameters = _get_function_parameters(function) name = function.name tags.append( TagFormat( begin=f"{TOOL_CALL_BEGIN}function{TOOL_SEP}{name}{JSON_RENDER_BEGIN}", content=JSONSchemaFormat(json_schema=parameters, any_order=any_order), end=f"{JSON_RENDER_END}{TOOL_CALL_END}", ) ) if len(tags) > 0: inner_tool_calls = TagsWithSeparatorFormat(tags=tags, separator="\n", at_least_one=True) tool_calls = TagFormat( begin=TOOL_CALLS_BEGIN, content=inner_tool_calls, end=TOOL_CALLS_END ) suffix_tag = TriggeredTagsFormat( triggers=[TOOL_CALLS_BEGIN], tags=[tool_calls], excludes=_text_excludes(exclude_special_tokens, THINK_EXCLUDE_TOKENS), ) else: suffix_tag = AnyTextFormat( excludes=_text_excludes(exclude_special_tokens, THINK_EXCLUDE_TOKENS) ) elif tool_choice == "forced": if not tools: raise ValueError("Forced tool choice must resolve to exactly one tool.") function = tools[0].function parameters = _get_function_parameters(function) suffix_tag = TagFormat( begin=f"{TOOL_CALLS_BEGIN}{TOOL_CALL_BEGIN}function{TOOL_SEP}{function.name}{JSON_RENDER_BEGIN}", content=JSONSchemaFormat(json_schema=parameters, any_order=any_order), end=f"{JSON_RENDER_END}{TOOL_CALL_END}{TOOL_CALLS_END}", ) elif tool_choice == "required": tags = [] for tool in tools: function = tool.function parameters = _get_function_parameters(function) name = function.name tags.append( TagFormat( begin=f"{TOOL_CALL_BEGIN}function{TOOL_SEP}{name}{JSON_RENDER_BEGIN}", content=JSONSchemaFormat(json_schema=parameters, any_order=any_order), end=f"{JSON_RENDER_END}{TOOL_CALL_END}", ) ) assert len(tags) > 0 inner_tool_calls = TagsWithSeparatorFormat(tags=tags, separator="\n", at_least_one=True) suffix_tag = TagFormat(begin=TOOL_CALLS_BEGIN, content=inner_tool_calls, end=TOOL_CALLS_END) if not reasoning: return StructuralTag(format=suffix_tag) prefix_tag = TagFormat(begin="", content=AnyTextFormat(), end=THINK_TAG_END) return StructuralTag(format=SequenceFormat(elements=[prefix_tag, suffix_tag])) @register_model_structural_tag("deepseek_v3_1") def get_deepseek_v3_1_structural_tag( tools: Optional[List[FunctionToolParam]] = None, builtin_tools: Optional[List[BuiltinToolParam]] = None, tool_choice: Literal["auto", "required", "forced"] = "auto", reasoning: bool = True, any_order: bool = False, exclude_special_tokens: bool = True, **kwargs: Any, ) -> StructuralTag: """Get DeepSeek-V3.1 style structural tag format. Corresponding model key: ``"deepseek_v3_1"``. Reference: https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/main/tokenizer_config.json Supported models: - DeepSeek-V3.1 - DeepSeek-V3.2-Exp """ TOOL_CALLS_BEGIN = "<｜tool▁calls▁begin｜>" TOOL_CALLS_END = "<｜tool▁calls▁end｜>" TOOL_CALL_BEGIN = "<｜tool▁call▁begin｜>" TOOL_CALL_END = "<｜tool▁call▁end｜>" TOOL_SEP = "<｜tool▁sep｜>" THINK_TAG_END = "" THINK_EXCLUDE_TOKENS = ["", ""] tools = tools or [] builtin_tools = builtin_tools or [] if tool_choice == "auto": tags = [] for tool in tools: function = tool.function parameters = _get_function_parameters(function) name = function.name tags.append( TagFormat( begin=f"{TOOL_CALL_BEGIN}{name}{TOOL_SEP}", content=JSONSchemaFormat(json_schema=parameters, any_order=any_order), end=TOOL_CALL_END, ) ) if len(tags) > 0: inner_tool_calls = TagsWithSeparatorFormat(tags=tags, separator="", at_least_one=True) tool_calls = TagFormat( begin=TOOL_CALLS_BEGIN, content=inner_tool_calls, end=TOOL_CALLS_END ) suffix_tag = TriggeredTagsFormat( triggers=[TOOL_CALLS_BEGIN], tags=[tool_calls], excludes=_text_excludes(exclude_special_tokens, THINK_EXCLUDE_TOKENS), ) else: suffix_tag = AnyTextFormat( excludes=_text_excludes(exclude_special_tokens, THINK_EXCLUDE_TOKENS) ) elif tool_choice == "forced": if not tools: raise ValueError("Forced tool choice must resolve to exactly one tool.") function = tools[0].function parameters = _get_function_parameters(function) suffix_tag = TagFormat( begin=f"{TOOL_CALLS_BEGIN}{TOOL_CALL_BEGIN}{function.name}{TOOL_SEP}", content=JSONSchemaFormat(json_schema=parameters, any_order=any_order), end=f"{TOOL_CALL_END}{TOOL_CALLS_END}", ) elif tool_choice == "required": tags = [] for tool in tools: function = tool.function parameters = _get_function_parameters(function) name = function.name tags.append( TagFormat( begin=f"{TOOL_CALL_BEGIN}{name}{TOOL_SEP}", content=JSONSchemaFormat(json_schema=parameters, any_order=any_order), end=TOOL_CALL_END, ) ) assert len(tags) > 0 inner_tool_calls = TagsWithSeparatorFormat(tags=tags, separator="", at_least_one=True) suffix_tag = TagFormat(begin=TOOL_CALLS_BEGIN, content=inner_tool_calls, end=TOOL_CALLS_END) if not reasoning: return StructuralTag(format=suffix_tag) prefix_tag = TagFormat(begin="", content=AnyTextFormat(), end=THINK_TAG_END) return StructuralTag(format=SequenceFormat(elements=[prefix_tag, suffix_tag])) @register_model_structural_tag("qwen_3_5") @register_model_structural_tag("qwen_3_coder") def get_qwen_3_5_structural_tag( tools: Optional[List[FunctionToolParam]] = None, builtin_tools: Optional[List[BuiltinToolParam]] = None, tool_choice: Literal["auto", "required", "forced"] = "auto", reasoning: bool = True, any_order: bool = False, exclude_special_tokens: bool = True, **kwargs: Any, ) -> StructuralTag: """Get Qwen XML tool-call structural tag format. Corresponding model keys: ``"qwen_3_5"`` and ``"qwen_3_coder"``. Reference: https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8/blob/main/chat_template.jinja Parameters are normalized by :func:`get_model_structural_tag` before this function is called: - ``tools``: a list of function tools. Each tool should have a ``function`` object containing ``name`` and ``parameters`` fields. - ``reasoning``: whether to add the ```` reasoning prefix before the tool/text suffix. Supported models: - Qwen3.5 - Qwen3.6 - Qwen3-Coder - Qwen3-Coder-Next Returns ------- StructuralTag A structural tag for Qwen XML function calling format. """ TOOL_CALL_BEGIN_PREFIX = "\n", ""] tools = tools or [] builtin_tools = builtin_tools or [] if tool_choice == "auto": tags = [] for tool in tools: function = tool.function parameters = _get_function_parameters(function) name = function.name tags.append( TagFormat( begin=f"{TOOL_CALL_BEGIN_PREFIX}{name}{TOOL_CALL_BEGIN_SUFFIX}", content=JSONSchemaFormat( json_schema=parameters, style="qwen_xml", any_order=any_order ), end=TOOL_CALL_END, ) ) if len(tags) > 0: suffix_tag = TriggeredTagsFormat( triggers=[TOOL_CALL_TRIGGER], tags=tags, excludes=_text_excludes(exclude_special_tokens, THINK_EXCLUDE_TOKENS), ) else: suffix_tag = AnyTextFormat( excludes=_text_excludes(exclude_special_tokens, THINK_EXCLUDE_TOKENS) ) elif tool_choice == "forced": if not tools: raise ValueError("Forced tool choice must resolve to exactly one tool.") function = tools[0].function suffix_tag = TagFormat( begin=f"{TOOL_CALL_BEGIN_PREFIX}{function.name}{TOOL_CALL_BEGIN_SUFFIX}", content=JSONSchemaFormat( json_schema=_get_function_parameters(function), style="qwen_xml", any_order=any_order, ), end=TOOL_CALL_END, ) elif tool_choice == "required": tags = [] for tool in tools: function = tool.function parameters = _get_function_parameters(function) name = function.name tags.append( TagFormat( begin=f"{TOOL_CALL_BEGIN_PREFIX}{name}{TOOL_CALL_BEGIN_SUFFIX}", content=JSONSchemaFormat( json_schema=parameters, style="qwen_xml", any_order=any_order ), end=TOOL_CALL_END, ) ) assert len(tags) > 0 suffix_tag = TagsWithSeparatorFormat(tags=tags, separator="\n", at_least_one=True) if not reasoning: return StructuralTag(format=suffix_tag) prefix_tag = SequenceFormat( elements=[ TagFormat(begin="", content=AnyTextFormat(), end=THINK_TAG_END), ConstStringFormat(value=THINK_SUFFIX), ] ) return StructuralTag(format=SequenceFormat(elements=[prefix_tag, suffix_tag])) get_qwen_3_coder_structural_tag = get_qwen_3_5_structural_tag """Deprecated alias for :func:`get_qwen_3_5_structural_tag`.""" @register_model_structural_tag("qwen_3") def get_qwen_3_structural_tag( tools: Optional[List[FunctionToolParam]] = None, builtin_tools: Optional[List[BuiltinToolParam]] = None, tool_choice: Literal["auto", "required", "forced"] = "auto", reasoning: bool = True, any_order: bool = False, exclude_special_tokens: bool = True, **kwargs: Any, ) -> StructuralTag: """Get Qwen3 style structural tag format. Corresponding model key: ``"qwen_3"``. Reference: https://qwen.readthedocs.io/en/latest/framework/function_call.html Parameters are normalized by :func:`get_model_structural_tag` before this function is called: - ``tools``: a list of function tools. Each tool should have a ``function`` object containing ``name`` and ``parameters`` fields. - ``reasoning``: whether to enable reasoning mode. If ``False``, remove the reasoning part. Supported models: - Qwen3 - Qwen3-Next Returns ------- StructuralTag A structural tag template. This format is used by Qwen3 and other models that follow the same style. """ TOOL_CALL_BEGIN_PREFIX = '\n{"name": "' ARGUMENTS_FIELD_PREFIX = '", "arguments": ' TOOL_CALL_END = "}\n" TOOL_CALL_TRIGGER = "" THINK_TAG_END = "" THINK_SUFFIX = "\n\n" THINK_EXCLUDE_TOKENS = ["", ""] tools = tools or [] builtin_tools = builtin_tools or [] if tool_choice == "auto": tags = [] for tool in tools: function = tool.function parameters = _get_function_parameters(function) name = function.name tags.append( TagFormat( begin=(TOOL_CALL_BEGIN_PREFIX + name + ARGUMENTS_FIELD_PREFIX), content=JSONSchemaFormat(json_schema=parameters, any_order=any_order), end=TOOL_CALL_END, ) ) if len(tags) > 0: suffix_tag = TriggeredTagsFormat( triggers=[TOOL_CALL_TRIGGER], tags=tags, excludes=_text_excludes(exclude_special_tokens, THINK_EXCLUDE_TOKENS), ) else: suffix_tag = AnyTextFormat( excludes=_text_excludes(exclude_special_tokens, THINK_EXCLUDE_TOKENS) ) elif tool_choice == "forced": if not tools: raise ValueError("Forced tool choice must resolve to exactly one tool.") function = tools[0].function suffix_tag = TagFormat( begin=(TOOL_CALL_BEGIN_PREFIX + function.name + ARGUMENTS_FIELD_PREFIX), content=JSONSchemaFormat( json_schema=_get_function_parameters(function), any_order=any_order ), end=TOOL_CALL_END, ) elif tool_choice == "required": tags = [] for tool in tools: function = tool.function parameters = _get_function_parameters(function) name = function.name tags.append( TagFormat( begin=(TOOL_CALL_BEGIN_PREFIX + name + ARGUMENTS_FIELD_PREFIX), content=JSONSchemaFormat(json_schema=parameters, any_order=any_order), end=TOOL_CALL_END, ) ) assert len(tags) > 0 suffix_tag = TagsWithSeparatorFormat(tags=tags, separator="\n", at_least_one=True) if not reasoning: return StructuralTag(format=suffix_tag) prefix_tag = SequenceFormat( elements=[ TagFormat(begin="", content=AnyTextFormat(), end=THINK_TAG_END), ConstStringFormat(value=THINK_SUFFIX), ] ) return StructuralTag(format=SequenceFormat(elements=[prefix_tag, suffix_tag])) @register_model_structural_tag("harmony") def get_harmony_structural_tag( tools: Optional[List[FunctionToolParam]] = None, builtin_tools: Optional[List[BuiltinToolParam]] = None, tool_choice: Literal["auto", "required", "forced"] = "auto", reasoning: bool = True, any_order: bool = False, exclude_special_tokens: bool = True, **kwargs: Any, ) -> StructuralTag: """Get harmony(gpt-oss) style structural tag format. Corresponding model key: ``"harmony"``. Reference: https://developers.openai.com/cookbook/articles/openai-harmony Reference: https://huggingface.co/openai/gpt-oss-120b/blob/main/chat_template.jinja Parameters are normalized by :func:`get_model_structural_tag` before this function is called: - ``tools``: a list of function tools. Each tool should have a ``function`` object containing ``name`` and ``parameters`` fields. - ``builtin_tools``: a list of builtin tools. Each builtin tool should provide ``type``, optional ``name``, and ``parameters`` fields. - ``reasoning``: whether to enable the analysis channel. Supported models: - gpt-oss Returns ------- StructuralTag A structural tag template. This format is in OpenAI Harmony Response Format, which is used by GPT-oss and other models that follow the same style. """ CALL_END = "<|call|>" FINAL_BEGIN = "<|channel|>final<|message|>" FINAL_END = ["<|end|>", "<|return|>"] ANALYSIS_BEGIN = "<|channel|>analysis<|message|>" TAG_SEPARATOR = "<|start|>assistant" def _function_tool_tags(name, parameters): """Generate tags for all supported harmony function tool call formats.""" content = JSONSchemaFormat(json_schema=parameters, any_order=any_order) return [ TagFormat( begin=f"<|channel|>commentary to=functions.{name}<|constrain|>json<|message|>", content=content, end=CALL_END, ), TagFormat( begin=f" to=functions.{name}<|channel|>commentary <|constrain|>json<|message|>", content=content, end=CALL_END, ), TagFormat( begin=f" to=functions.{name}<|channel|>commentary json<|message|>", content=content, end=CALL_END, ), ] def _builtin_tool_tags(name, parameters): """Generate tags for supported harmony builtin tool call formats.""" content = JSONSchemaFormat(json_schema=parameters, any_order=any_order) return [ TagFormat( begin=f"<|channel|>commentary to={name} code<|message|>", content=content, end=CALL_END, ), TagFormat( begin=f" to={name}<|channel|>commentary code<|message|>", content=content, end=CALL_END, ), ] tools = tools or [] builtin_tools = builtin_tools or [] tags = [] if tool_choice == "auto": for tool in tools: function = tool.function parameters = _get_function_parameters(function) tags.extend(_function_tool_tags(function.name, parameters)) for tool in builtin_tools: parameters = _get_function_parameters(tool) name = _get_builtin_tool_name(tool) tags.extend(_builtin_tool_tags(name, parameters)) final_tag = TagFormat(begin=FINAL_BEGIN, content=AnyTextFormat(), end=FINAL_END) tags.append(final_tag) elif tool_choice == "forced": if builtin_tools: tags.extend( _builtin_tool_tags( _get_builtin_tool_name(builtin_tools[0]), _get_function_parameters(builtin_tools[0]), ) ) elif tools: function = tools[0].function tags.extend(_function_tool_tags(function.name, _get_function_parameters(function))) else: raise ValueError("Forced tool choice must resolve to exactly one tool.") elif tool_choice == "required": for tool in builtin_tools: parameters = _get_function_parameters(tool) name = _get_builtin_tool_name(tool) tags.extend(_builtin_tool_tags(name, parameters)) for tool in tools: function = tool.function parameters = _get_function_parameters(function) tags.extend(_function_tool_tags(function.name, parameters)) assert len(tags) > 0 if reasoning: analysis_tag = TagFormat(begin=ANALYSIS_BEGIN, content=AnyTextFormat(), end=FINAL_END) tags.append(analysis_tag) tags_with_separator = TagsWithSeparatorFormat(tags=tags, separator=TAG_SEPARATOR) return StructuralTag(format=tags_with_separator) @register_model_structural_tag("deepseek_v3_2") def get_deepseek_v3_2_structural_tag( tools: Optional[List[FunctionToolParam]] = None, builtin_tools: Optional[List[BuiltinToolParam]] = None, tool_choice: Literal["auto", "required", "forced"] = "auto", reasoning: bool = True, any_order: bool = False, exclude_special_tokens: bool = True, **kwargs: Any, ) -> StructuralTag: """Get DeepSeek-V3.2 style structural tag format. Corresponding model key: ``"deepseek_v3_2"``. Supported models: - DeepSeek-V3.2 """ INVOKE_BEGIN_PREFIX = '<｜DSML｜invoke name="' INVOKE_BEGIN_SUFFIX = '">\n' # INVOKE_END keeps a trailing "\n" so the final invoke is followed by a # single "\n" before , matching the official # DeepSeek-V3.2 chat template. The separator between consecutive invokes # is intentionally empty: the chat template joins tool calls with a single # "\n" and that "\n" is already supplied by INVOKE_END. INVOKE_END = "\n" INVOKE_SEPARATOR = "" TOOL_CALLS_PREFIX = "\n\n" FUNCTION_CALLS_BEGIN = "<｜DSML｜function_calls>\n" FUNCTION_CALLS_END = "" FUNCTION_CALLS_TRIGGER = "<｜DSML｜function_calls>" THINK_TAG_END = "" THINK_EXCLUDE_TOKENS = ["", ""] XML_STYLE = "deepseek_xml" tools = tools or [] builtin_tools = builtin_tools or [] if tool_choice == "auto": tags = [] for tool in tools: function = tool.function parameters = _get_function_parameters(function) name = function.name tags.append( TagFormat( begin=(INVOKE_BEGIN_PREFIX + name + INVOKE_BEGIN_SUFFIX), content=JSONSchemaFormat( json_schema=parameters, style=XML_STYLE, any_order=any_order ), end=INVOKE_END, ) ) # generate function calling triggered tag if len(tags) > 0: function_calling_tags = TagsWithSeparatorFormat( tags=tags, separator=INVOKE_SEPARATOR, at_least_one=True ) suffix_tag = TriggeredTagsFormat( triggers=[FUNCTION_CALLS_TRIGGER], tags=[ TagFormat( begin=FUNCTION_CALLS_BEGIN, content=function_calling_tags, end=FUNCTION_CALLS_END, ) ], excludes=_text_excludes(exclude_special_tokens, THINK_EXCLUDE_TOKENS), ) else: suffix_tag = AnyTextFormat( excludes=_text_excludes(exclude_special_tokens, THINK_EXCLUDE_TOKENS) ) elif tool_choice == "forced": if not tools: raise ValueError("Forced tool choice must resolve to exactly one tool.") function = tools[0].function suffix_tag = SequenceFormat( elements=[ ConstStringFormat(value=TOOL_CALLS_PREFIX + FUNCTION_CALLS_BEGIN), TagFormat( begin=(INVOKE_BEGIN_PREFIX + function.name + INVOKE_BEGIN_SUFFIX), content=JSONSchemaFormat( json_schema=_get_function_parameters(function), style=XML_STYLE, any_order=any_order, ), end=INVOKE_END, ), ConstStringFormat(value=FUNCTION_CALLS_END), ] ) elif tool_choice == "required": tags = [] for tool in tools: function = tool.function parameters = _get_function_parameters(function) name = function.name tags.append( TagFormat( begin=(INVOKE_BEGIN_PREFIX + name + INVOKE_BEGIN_SUFFIX), content=JSONSchemaFormat( json_schema=parameters, style=XML_STYLE, any_order=any_order ), end=INVOKE_END, ) ) assert len(tags) > 0 suffix_tag = SequenceFormat( elements=[ ConstStringFormat(value=TOOL_CALLS_PREFIX + FUNCTION_CALLS_BEGIN), TagsWithSeparatorFormat(tags=tags, separator=INVOKE_SEPARATOR, at_least_one=True), ConstStringFormat(value=FUNCTION_CALLS_END), ] ) if not reasoning: return StructuralTag(format=suffix_tag) prefix_tag = TagFormat(begin="", content=AnyTextFormat(), end=THINK_TAG_END) sequence_format = SequenceFormat(elements=[prefix_tag, suffix_tag]) return StructuralTag(format=sequence_format) @register_model_structural_tag("minimax") def get_minimax_structural_tag( tools: Optional[List[FunctionToolParam]] = None, builtin_tools: Optional[List[BuiltinToolParam]] = None, tool_choice: Literal["auto", "required", "forced"] = "auto", reasoning: bool = True, any_order: bool = False, exclude_special_tokens: bool = True, **kwargs: Any, ) -> StructuralTag: """Get MiniMax-M2.5 style structural tag format. Corresponding model key: ``"minimax"``. Supported models: - MiniMax-M2.5 - MiniMax-M2.7 Returns ------- StructuralTag A structural tag for MiniMax function calling format. """ INVOKE_BEGIN_PREFIX = '\n' INVOKE_END = "\n" TOOL_CALL_BEGIN = "\n" TOOL_CALL_END = "" TOOL_CALL_TRIGGER = "" THINK_TAG_END = "" THINK_SUFFIX = "\n\n" EMPTY_THINK_CONTENT = "\n\n\n" THINK_EXCLUDE_TOKENS = ["", ""] XML_STYLE = "minimax_xml" tools = tools or [] builtin_tools = builtin_tools or [] if tool_choice == "auto": tags = [] for tool in tools: function = tool.function parameters = _get_function_parameters(function) name = function.name tags.append( TagFormat( begin=(INVOKE_BEGIN_PREFIX + name + INVOKE_BEGIN_SUFFIX), content=JSONSchemaFormat( json_schema=parameters, style=XML_STYLE, any_order=any_order ), end=INVOKE_END, ) ) # generate function calling triggered tag if len(tags) > 0: function_calling_tags = TagsWithSeparatorFormat( tags=tags, separator="", at_least_one=True ) suffix_tag = TriggeredTagsFormat( triggers=[TOOL_CALL_TRIGGER], tags=[ TagFormat( begin=TOOL_CALL_BEGIN, content=function_calling_tags, end=TOOL_CALL_END ) ], excludes=_text_excludes(exclude_special_tokens, THINK_EXCLUDE_TOKENS), ) else: suffix_tag = AnyTextFormat( excludes=_text_excludes(exclude_special_tokens, THINK_EXCLUDE_TOKENS) ) elif tool_choice == "forced": if not tools: raise ValueError("Forced tool choice must resolve to exactly one tool.") function = tools[0].function suffix_tag = SequenceFormat( elements=[ ConstStringFormat(value="\n" + TOOL_CALL_BEGIN), TagFormat( begin=(INVOKE_BEGIN_PREFIX + function.name + INVOKE_BEGIN_SUFFIX), content=JSONSchemaFormat( json_schema=_get_function_parameters(function), style=XML_STYLE, any_order=any_order, ), end=INVOKE_END, ), ConstStringFormat(value=TOOL_CALL_END), ] ) elif tool_choice == "required": tags = [] for tool in tools: function = tool.function parameters = _get_function_parameters(function) name = function.name tags.append( TagFormat( begin=(INVOKE_BEGIN_PREFIX + name + INVOKE_BEGIN_SUFFIX), content=JSONSchemaFormat( json_schema=parameters, style=XML_STYLE, any_order=any_order ), end=INVOKE_END, ) ) assert len(tags) > 0 suffix_tag = SequenceFormat( elements=[ ConstStringFormat(value="\n" + TOOL_CALL_BEGIN), TagsWithSeparatorFormat(tags=tags, separator="", at_least_one=True), ConstStringFormat(value=TOOL_CALL_END), ] ) if reasoning: think_tag = TagFormat(begin="", content=AnyTextFormat(), end=THINK_TAG_END) else: think_tag = ConstStringFormat(value=EMPTY_THINK_CONTENT) return StructuralTag( format=SequenceFormat( elements=[think_tag, ConstStringFormat(value=THINK_SUFFIX), suffix_tag] ) ) @register_model_structural_tag("glm_4_7") def get_glm_4_7_structural_tag( tools: Optional[List[FunctionToolParam]] = None, builtin_tools: Optional[List[BuiltinToolParam]] = None, tool_choice: Literal["auto", "required", "forced"] = "auto", reasoning: bool = True, any_order: bool = False, exclude_special_tokens: bool = True, **kwargs: Any, ) -> StructuralTag: """Get GLM-4.7/GLM-5 style structural tag format. The GLM tool calling format uses XML-like tags: ``function_name`` ``keyvalue`` ```` Corresponding model key: ``"glm_4_7"``. Parameters are normalized by :func:`get_model_structural_tag` before this function is called: - ``tools``: a list of function tools. Each tool should have a ``function`` object containing ``name`` and ``parameters`` fields. - ``reasoning``: whether to enable reasoning mode. If ``False``, use the non-reasoning mode. Supported models: - GLM-5 - GLM-4.7 Returns ------- StructuralTag A structural tag for GLM function calling format. """ TOOL_CALL_BEGIN_PREFIX = "" TOOL_CALL_END = "" TOOL_CALL_TRIGGER = "" THINK_TAG_END = "" THINK_EXCLUDE_TOKENS = ["", ""] XML_STYLE = "glm_xml" # GLM tool-call control tokens are reserved special tokens that are only # valid inside a tool call. They must never appear in reasoning or free-form # text, otherwise the model can emit a stray control token that downstream # parsers mis-interpret. Exclude them from every free-text region. ARG_TOKENS = ["", "", "", ""] # Reasoning contains no tool calls at all -> exclude every control token. REASONING_EXCLUDES = THINK_EXCLUDE_TOKENS + [TOOL_CALL_BEGIN_PREFIX, TOOL_CALL_END] + ARG_TOKENS # Free text after may *start* a tool call via the # trigger, so that trigger stays allowed; every other control token is not. TEXT_EXCLUDES = THINK_EXCLUDE_TOKENS + [TOOL_CALL_END] + ARG_TOKENS tools = tools or [] builtin_tools = builtin_tools or [] if tool_choice == "auto": tags = [] for tool in tools: function = tool.function parameters = _get_function_parameters(function) name = function.name tags.append( TagFormat( begin=f"{TOOL_CALL_BEGIN_PREFIX}{name}", content=JSONSchemaFormat( json_schema=parameters, style=XML_STYLE, any_order=any_order ), end=TOOL_CALL_END, ) ) if len(tags) > 0: suffix_tag = TriggeredTagsFormat( triggers=[TOOL_CALL_TRIGGER], tags=tags, excludes=_text_excludes(exclude_special_tokens, TEXT_EXCLUDES), ) else: suffix_tag = AnyTextFormat( excludes=_text_excludes(exclude_special_tokens, REASONING_EXCLUDES) ) elif tool_choice == "forced": if not tools: raise ValueError("Forced tool choice must resolve to exactly one tool.") function = tools[0].function suffix_tag = TagFormat( begin=f"{TOOL_CALL_BEGIN_PREFIX}{function.name}", content=JSONSchemaFormat( json_schema=_get_function_parameters(function), style=XML_STYLE, any_order=any_order ), end=TOOL_CALL_END, ) elif tool_choice == "required": tags = [] for tool in tools: function = tool.function parameters = _get_function_parameters(function) name = function.name tags.append( TagFormat( begin=f"{TOOL_CALL_BEGIN_PREFIX}{name}", content=JSONSchemaFormat( json_schema=parameters, style=XML_STYLE, any_order=any_order ), end=TOOL_CALL_END, ) ) assert len(tags) > 0 suffix_tag = TagsWithSeparatorFormat(tags=tags, separator="", at_least_one=True) if not reasoning: return StructuralTag(format=suffix_tag) prefix_tag = TagFormat( begin="", content=AnyTextFormat(excludes=_text_excludes(exclude_special_tokens, REASONING_EXCLUDES)), end=THINK_TAG_END, ) return StructuralTag(format=SequenceFormat(elements=[prefix_tag, suffix_tag])) # TODO: We are dropping Gemma support because its parameter format is special and not supported # yet: the string are wrapped by <|"|> instead of ". We will support it later and get it back. # @register_model_structural_tag("gemma_4") def _get_gemma_4_structural_tag( tools: Optional[List[FunctionToolParam]] = None, builtin_tools: Optional[List[BuiltinToolParam]] = None, tool_choice: Literal["auto", "required", "forced"] = "auto", reasoning: bool = True, any_order: bool = False, exclude_special_tokens: bool = True, **kwargs: Any, ) -> StructuralTag: """Get Gemma 4 style structural tag format. Gemma 4 uses channel markers for reasoning and tool calls instead of ````/````: - Thinking: ``<|channel>thought\\n...thinking...`` - Tool calls: ``<|tool_call>call:func_name{...}`` - Turn end: ```` Corresponding model key: ``"gemma_4"``. Reference: https://ai.google.dev/gemma/docs/core/prompt-formatting-gemma4 Parameters are normalized by :func:`get_model_structural_tag` before this function is called: - ``tools``: a list of function tools. Each tool should have a ``function`` object containing ``name`` and ``parameters`` fields. - ``reasoning``: whether to enable reasoning mode. If ``False``, the reasoning channel is omitted. - ``tool_choice``: ``"auto"`` or ``"required"``. ``"required"`` forces at least one tool call. Supported models: - Gemma-4 - gemma-4-12b-it - gemma-4-26b-a4b-it - gemma-4-31b-it - gemma-4-e2b-it Returns ------- StructuralTag A structural tag for Gemma 4 function calling format. """ TOOL_CALL_BEGIN_PREFIX = "<|tool_call>call:" TOOL_CALL_END = "" TOOL_CALL_TRIGGER = "<|tool_call>" THINK_TAG_BEGIN = "<|channel>thought\n" THINK_TAG_END = "" GEMMA4_EXCLUDE_TOKENS = ["<|channel>", ""] tools = tools or [] builtin_tools = builtin_tools or [] if tool_choice == "auto": tags = [] for tool in tools: function = tool.function parameters = _get_function_parameters(function) name = function.name tags.append( TagFormat( begin=TOOL_CALL_BEGIN_PREFIX + name, content=JSONSchemaFormat(json_schema=parameters, any_order=any_order), end=TOOL_CALL_END, ) ) if len(tags) > 0: suffix_tag = TriggeredTagsFormat( triggers=[TOOL_CALL_TRIGGER], tags=tags, excludes=_text_excludes(exclude_special_tokens, GEMMA4_EXCLUDE_TOKENS), ) else: suffix_tag = AnyTextFormat( excludes=_text_excludes(exclude_special_tokens, GEMMA4_EXCLUDE_TOKENS) ) elif tool_choice == "forced": if not tools: raise ValueError("Forced tool choice must resolve to exactly one tool.") function = tools[0].function suffix_tag = TagFormat( begin=TOOL_CALL_BEGIN_PREFIX + function.name, content=JSONSchemaFormat( json_schema=_get_function_parameters(function), any_order=any_order ), end=TOOL_CALL_END, ) elif tool_choice == "required": tags = [] for tool in tools: function = tool.function parameters = _get_function_parameters(function) name = function.name tags.append( TagFormat( begin=TOOL_CALL_BEGIN_PREFIX + name, content=JSONSchemaFormat(json_schema=parameters, any_order=any_order), end=TOOL_CALL_END, ) ) assert len(tags) > 0 suffix_tag = TagsWithSeparatorFormat(tags=tags, separator="", at_least_one=True) if not reasoning: return StructuralTag(format=suffix_tag) prefix_tag = TagFormat(begin=THINK_TAG_BEGIN, content=AnyTextFormat(), end=THINK_TAG_END) return StructuralTag(format=SequenceFormat(elements=[prefix_tag, suffix_tag])) @register_model_structural_tag("deepseek_v4") def get_deepseek_v4_structural_tag( tools: Optional[List[FunctionToolParam]] = None, builtin_tools: Optional[List[BuiltinToolParam]] = None, tool_choice: Literal["auto", "required", "forced"] = "auto", reasoning: bool = True, any_order: bool = False, exclude_special_tokens: bool = True, **kwargs: Any, ) -> StructuralTag: """Get DeepSeek-V4 style structural tag format. Corresponding model key: ``"deepseek_v4"``. Supported models: - DeepSeek-V4 """ INVOKE_BEGIN_PREFIX = '<｜DSML｜invoke name="' INVOKE_BEGIN_SUFFIX = '">\n' # See get_deepseek_v3_2_structural_tag for the rationale on INVOKE_END + # INVOKE_SEPARATOR splitting the single "\n" join that the chat template # uses between consecutive <｜DSML｜invoke> blocks. INVOKE_END = "\n" INVOKE_SEPARATOR = "" TOOL_CALLS_PREFIX = "\n\n" FUNCTION_CALLS_BEGIN = "<｜DSML｜tool_calls>\n" FUNCTION_CALLS_END = "" FUNCTION_CALLS_TRIGGER = "<｜DSML｜tool_calls>" THINK_TAG_END = "" THINK_EXCLUDE_TOKENS = ["", ""] XML_STYLE = "deepseek_xml" tools = tools or [] builtin_tools = builtin_tools or [] if tool_choice == "auto": tags = [] for tool in tools: function = tool.function parameters = _get_function_parameters(function) name = function.name tags.append( TagFormat( begin=(INVOKE_BEGIN_PREFIX + name + INVOKE_BEGIN_SUFFIX), content=JSONSchemaFormat( json_schema=parameters, style=XML_STYLE, any_order=any_order ), end=INVOKE_END, ) ) # generate function calling triggered tag if len(tags) > 0: function_calling_tags = TagsWithSeparatorFormat( tags=tags, separator=INVOKE_SEPARATOR, at_least_one=True ) suffix_tag = TriggeredTagsFormat( triggers=[FUNCTION_CALLS_TRIGGER], tags=[ TagFormat( begin=FUNCTION_CALLS_BEGIN, content=function_calling_tags, end=FUNCTION_CALLS_END, ) ], excludes=_text_excludes(exclude_special_tokens, THINK_EXCLUDE_TOKENS), ) else: suffix_tag = AnyTextFormat( excludes=_text_excludes(exclude_special_tokens, THINK_EXCLUDE_TOKENS) ) elif tool_choice == "forced": if not tools: raise ValueError("Forced tool choice must resolve to exactly one tool.") function = tools[0].function suffix_tag = SequenceFormat( elements=[ ConstStringFormat(value=TOOL_CALLS_PREFIX + FUNCTION_CALLS_BEGIN), TagFormat( begin=(INVOKE_BEGIN_PREFIX + function.name + INVOKE_BEGIN_SUFFIX), content=JSONSchemaFormat( json_schema=_get_function_parameters(function), style=XML_STYLE, any_order=any_order, ), end=INVOKE_END, ), ConstStringFormat(value=FUNCTION_CALLS_END), ] ) elif tool_choice == "required": tags = [] for tool in tools: function = tool.function parameters = _get_function_parameters(function) name = function.name tags.append( TagFormat( begin=(INVOKE_BEGIN_PREFIX + name + INVOKE_BEGIN_SUFFIX), content=JSONSchemaFormat( json_schema=parameters, style=XML_STYLE, any_order=any_order ), end=INVOKE_END, ) ) assert len(tags) > 0 suffix_tag = SequenceFormat( elements=[ ConstStringFormat(value=TOOL_CALLS_PREFIX + FUNCTION_CALLS_BEGIN), TagsWithSeparatorFormat(tags=tags, separator=INVOKE_SEPARATOR, at_least_one=True), ConstStringFormat(value=FUNCTION_CALLS_END), ] ) if not reasoning: return StructuralTag(format=suffix_tag) prefix_tag = TagFormat(begin="", content=AnyTextFormat(), end=THINK_TAG_END) sequence_format = SequenceFormat(elements=[prefix_tag, suffix_tag]) return StructuralTag(format=sequence_format) # Backward-compatible alias get_builtin_structural_tag = get_model_structural_tag """Alias for :func:`get_model_structural_tag`. Deprecated.""" xgrammar-0.2.3/python/xgrammar/compiler.py000066400000000000000000000312451521764210300206630ustar00rootroot00000000000000"""Compiling grammar for efficient token mask generation.""" from typing import Any, Dict, List, Optional, Tuple, Type, Union, overload from pydantic import BaseModel from typing_extensions import deprecated from .base import XGRObject, _core from .grammar import ( Grammar, StructuralTagItem, _convert_schema_to_str, _get_structural_tag_str_from_args, ) from .structural_tag import StructuralTag from .tokenizer_info import TokenizerInfo class CompiledGrammar(XGRObject): """This is the primary object to store compiled grammar. A CompiledGrammar can be used to construct GrammarMatcher to generate token masks efficiently. Notes ----- Do not construct this class directly, instead use :class:`GrammarCompiler` to construct the object. """ @property def grammar(self) -> Grammar: """The original grammar.""" return Grammar._create_from_handle(self._handle.grammar()) @property def tokenizer_info(self) -> TokenizerInfo: """The tokenizer info associated with the compiled grammar.""" return TokenizerInfo._create_from_handle(self._handle.tokenizer_info()) @property def memory_size_bytes(self) -> int: """The approximate memory usage of the compiled grammar in bytes.""" return self._handle.memory_size_bytes() def serialize_json(self) -> str: """Serialize the compiled grammar to a JSON string. It will serialize the compiled grammar without the tokenizer info, since the tokenizer info is shared by multiple compiled grammars. Notes ----- The metadata of the tokenizer info is serialized and will be checked when deserializing. Returns ------- json_string : str The JSON string. """ return str(self._handle.serialize_json()) @staticmethod def deserialize_json(json_str: str, tokenizer_info: TokenizerInfo) -> "CompiledGrammar": """Deserialize the compiled grammar from a JSON string and associate it with the specified tokenizer info. Notes ----- This will check the metadata of the tokenizer info matching the serialized metadata in json_str. If the metadata does not match, a DeserializeFormatError will be raised. Parameters ---------- json_str : str The JSON string. tokenizer_info : TokenizerInfo The tokenizer info. Returns ------- compiled_grammar : CompiledGrammar The compiled grammar. Raises ------ InvalidJSONError When the JSON string is invalid. DeserializeFormatError When the JSON string does not follow the serialization format of the grammar, or the tokenizer info metadata does not match. DeserializeVersionError When the __VERSION__ field in the JSON string is not the same as the current version. """ return CompiledGrammar._create_from_handle( _core.CompiledGrammar.deserialize_json(json_str, tokenizer_info._handle) ) class GrammarCompiler(XGRObject): """The compiler for grammars. It is associated with a certain tokenizer info, and compiles grammars into CompiledGrammar with the tokenizer info. It allows parallel compilation with multiple threads, and has a cache to store the compilation result, avoiding compiling the same grammar multiple times. """ def __init__( self, tokenizer_info: TokenizerInfo, *, max_threads: int = 8, cache_enabled: bool = True, cache_limit_bytes: int = -1, ): """Construct the compiler. Parameters ---------- tokenizer_info : TokenizerInfo The tokenizer info. max_threads : int, default: 8 The maximum number of threads used to compile the grammar. cache_enabled : bool, default: True Whether to enable the cache. cache_limit_bytes : int, default: -1 The maximum memory usage for the cache in the specified unit. Note that the actual memory usage may slightly exceed this value. """ if not isinstance(tokenizer_info, TokenizerInfo): raise ValueError( "Please convert the tokenizer to TokenizerInfo before passing it " "to GrammarCompiler." ) self._init_handle( _core.GrammarCompiler( tokenizer_info._handle, max_threads, cache_enabled, cache_limit_bytes ) ) def compile_json_schema( self, schema: Union[str, Type[BaseModel], Dict[str, Any]], *, any_whitespace: bool = True, indent: Optional[int] = None, separators: Optional[Tuple[str, str]] = None, strict_mode: bool = True, max_whitespace_cnt: Optional[int] = None, any_order: bool = False, ) -> CompiledGrammar: """Get CompiledGrammar from the specified JSON schema and format. The indent and separators parameters follow the same convention as in json.dumps(). Parameters ---------- schema : Union[str, Type[BaseModel], Dict[str, Any]] The schema string or Pydantic model or JSON schema dict. indent : Optional[int], default: None The number of spaces for indentation. If None, the output will be in one line. separators : Optional[Tuple[str, str]], default: None Two separators used in the schema: comma and colon. Examples: (",", ":"), (", ", ": "). If None, the default separators will be used: (",", ": ") when the indent is not None, and (", ", ": ") otherwise. strict_mode : bool, default: True Whether to use strict mode. In strict mode, the generated grammar will not allow properties and items that is not specified in the schema. This is equivalent to setting unevaluatedProperties and unevaluatedItems to false. This helps LLM to generate accurate output in the grammar-guided generation with JSON schema. max_whitespace_cnt : Optional[int], default: None The maximum number of whitespace characters allowed between elements, such like keys, values, separators and so on. If None, there is no limit on the number of whitespace characters. If specified, it will limit the number of whitespace characters to at most max_whitespace_cnt. It should be a positive integer. any_order : bool, default: False Whether object properties may appear in any order. - False: properties follow the schema's declared order, fully validated (required keys present, no duplicates). - True: properties may appear in any order; only key validity and each key's value schema are enforced. Key presence and uniqueness are not checked, so required keys may be missing and keys may repeat. The entry count is bounded to ``[max(minProperties, n_required), maxProperties]`` (unbounded when maxProperties is unset). Applies to every object, nested included. Returns ------- compiled_grammar : CompiledGrammar The compiled grammar. """ schema_str = _convert_schema_to_str(schema) return CompiledGrammar._create_from_handle( self._handle.compile_json_schema( schema_str, any_whitespace, indent, separators, strict_mode, max_whitespace_cnt, any_order, ) ) def compile_builtin_json_grammar(self) -> CompiledGrammar: """Get CompiledGrammar from the standard JSON. Returns ------- compiled_grammar : CompiledGrammar The compiled grammar. """ return CompiledGrammar._create_from_handle(self._handle.compile_builtin_json_grammar()) def compile_regex(self, regex: str) -> CompiledGrammar: """Get CompiledGrammar from the specified regex. Parameters ---------- regex : str The regex string. Returns ------- compiled_grammar : CompiledGrammar The compiled grammar. """ return CompiledGrammar._create_from_handle(self._handle.compile_regex(regex)) @overload def compile_structural_tag( self, structural_tag: Union[StructuralTag, str, Dict[str, Any]] ) -> CompiledGrammar: ... @overload @deprecated( "compile_structural_tag(tags, triggers) is deprecated. Compile structural tag with the " "StructuralTag class instead." ) def compile_structural_tag( self, tags: List[StructuralTagItem], triggers: List[str] ) -> CompiledGrammar: ... def compile_structural_tag(self, *args, **kwargs) -> CompiledGrammar: """Compile a grammar from a structural tag. See the Structural Tag Usage in XGrammar documentation for its usage. This method supports two calling patterns: 1. Single structural tag parameter: compile_structural_tag(structural_tag) 2. Legacy pattern (deprecated): compile_structural_tag(tags, triggers) Parameters ---------- structural_tag : Union[StructuralTag, str, Dict[str, Any]] The structural tag either as a StructuralTag object, or a JSON string or a dictionary. tags : List[StructuralTagItem] (Deprecated) The structural tags. Use StructuralTag class instead. triggers : List[str] (Deprecated) The triggers. Use StructuralTag class instead. Returns ------- compiled_grammar : CompiledGrammar The compiled grammar from the structural tag. Raises ------ InvalidJSONError When the structural tag is not a valid JSON string. InvalidStructuralTagError When the structural tag is not valid. TypeError When the arguments are invalid. Notes ----- The legacy pattern compile_structural_tag(tags, triggers) is deprecated. Use the StructuralTag class to construct structural tags instead. """ structural_tag_str = _get_structural_tag_str_from_args(args, kwargs) return CompiledGrammar._create_from_handle( self._handle.compile_structural_tag(structural_tag_str) ) @overload def compile_grammar( self, ebnf_string: str, *, root_rule_name: str = "root" ) -> CompiledGrammar: ... @overload def compile_grammar(self, grammar: Grammar) -> CompiledGrammar: ... def compile_grammar( self, grammar: Union[str, Grammar], *, root_rule_name: str = "root" ) -> CompiledGrammar: """Compile a grammar object. Overloads: 1. ``compile_grammar(ebnf_string: str, *, root_rule_name: str = "root") -> CompiledGrammar`` - Compile a grammar from an EBNF string. The string should follow the format described in https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md. 2. ``compile_grammar(grammar: Grammar) -> CompiledGrammar`` - Compile a grammar from a Grammar object. Parameters ---------- ebnf_string : str The grammar string in EBNF format. root_rule_name : str, default: "root" The name of the root rule in the grammar. grammar : Union[str, Grammar] The grammar string or Grammar object. Returns ------- compiled_grammar : CompiledGrammar The compiled grammar. """ if isinstance(grammar, str): return CompiledGrammar._create_from_handle( self._handle.compile_grammar_from_strings(grammar, root_rule_name) ) elif isinstance(grammar, Grammar): return CompiledGrammar._create_from_handle( self._handle.compile_grammar_ebnf(grammar._handle) ) else: raise ValueError("Invalid grammar type. Please pass a string or a Grammar object.") def clear_cache(self) -> None: """Clear all cached compiled grammars.""" self._handle.clear_cache() def get_cache_size_bytes(self) -> int: """The approximate memory usage of the cache in bytes.""" return self._handle.get_cache_size_bytes() @property def cache_limit_bytes(self) -> int: """ The maximum memory usage for the cache in bytes. Returns -1 if the cache has no memory limit. """ return self._handle.cache_limit_bytes() xgrammar-0.2.3/python/xgrammar/config.py000066400000000000000000000033051521764210300203120ustar00rootroot00000000000000"""Global configuration for XGrammar.""" from contextlib import contextmanager from .base import _core def get_max_recursion_depth() -> int: """Get the maximum allowed recursion depth. The depth is shared per process. The maximum recursion depth is determined in the following order: 1. Manually set via :py:func:`set_max_recursion_depth` 2. ``XGRAMMAR_MAX_RECURSION_DEPTH`` environment variable (if set and is a valid integer <= 1,000,000) 3. Default value of 10,000 Returns ------- max_recursion_depth : int The maximum allowed recursion depth. """ return _core.config.get_max_recursion_depth() def set_max_recursion_depth(max_recursion_depth: int) -> None: """Set the maximum allowed recursion depth. The depth is shared per process. This method is thread-safe. Parameters ---------- max_recursion_depth : int The maximum allowed recursion depth. """ _core.config.set_max_recursion_depth(max_recursion_depth) @contextmanager def max_recursion_depth(temp_depth: int): """A context manager for temporarily setting recursion depth. Examples -------- >>> with recursion_depth(1000): ... # recursion depth is 1000 here ... pass >>> # recursion depth is restored to original value """ prev_depth = get_max_recursion_depth() set_max_recursion_depth(temp_depth) try: yield finally: set_max_recursion_depth(prev_depth) def get_serialization_version() -> str: """Get the serialization version number. Returns ------- serialization_version : str The serialization version number. """ return str(_core.config.get_serialization_version()) xgrammar-0.2.3/python/xgrammar/contrib/000077500000000000000000000000001521764210300201325ustar00rootroot00000000000000xgrammar-0.2.3/python/xgrammar/contrib/__init__.py000066400000000000000000000000001521764210300222310ustar00rootroot00000000000000xgrammar-0.2.3/python/xgrammar/contrib/hf.py000066400000000000000000000110151521764210300210770ustar00rootroot00000000000000""" This file helps integrate xgrammar in HF transformers package by extending transformers.LogitsProcessor, which is to be fed to `model.generate()`. """ from typing import List, Union import torch import transformers import xgrammar as xgr class LogitsProcessor(transformers.LogitsProcessor): """ LogitsProcessor for processing logits in transformers' generate() method. Example usage ------------- .. code:: python model_name = "Qwen/Qwen2.5-0.5B-Instruct" tokenizer = AutoTokenizer.from_pretrained(model_name) config = AutoConfig.from_pretrained(model_name) # This can be larger than tokenizer.vocab_size due to paddings full_vocab_size = config.vocab_size tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=full_vocab_size) grammar_compiler = xgr.GrammarCompiler(tokenizer_info) compiled_grammar = grammar_compiler.compile_builtin_json_grammar() xgr_logits_processor = xgr.contrib.hf.LogitsProcessor(compiled_grammar) model.generate(prompt, logits_processor=[xgr_logits_processor]) For an end-to-end example, see folder `examples/hf_transformers/`. Notes ----- - Note that this LogitsProcessor can only be used once. For each `generate()` call, instantiate a new one. - Note that this implementation may contain extra overhead. """ def __init__(self, compiled_grammar: Union[xgr.CompiledGrammar, List[xgr.CompiledGrammar]]): """Initialize the LogitsProcessor. Parameters ---------- compiled_grammar : xgr.CompiledGrammar | List[xgr.CompiledGrammar] One or more grammars compiled according to the given grammar and the model's tokenizer_info. """ self.matchers: List[xgr.GrammarMatcher] = [] self.compiled_grammars: List[xgr.CompiledGrammar] = ( compiled_grammar if isinstance(compiled_grammar, list) else [compiled_grammar] ) self.full_vocab_size = self.compiled_grammars[0].tokenizer_info.vocab_size self.token_bitmask = None self.prefilled = False self.batch_size = 0 def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: """ Accept token sampled in the last iteration, fill in bitmask, and apply bitmask to logits. Returns: scores: Logits modified with bitmask. """ # Lazily initialize GrammarMatchers and bitmask if len(self.matchers) == 0: self.batch_size = input_ids.shape[0] self.compiled_grammars = ( self.compiled_grammars if len(self.compiled_grammars) > 1 else self.compiled_grammars * self.batch_size ) assert ( len(self.compiled_grammars) == self.batch_size ), "The number of compiled grammars must be equal to the batch size." self.matchers = [ xgr.GrammarMatcher(self.compiled_grammars[i]) for i in range(self.batch_size) ] self.token_bitmask = xgr.allocate_token_bitmask(self.batch_size, self.full_vocab_size) if input_ids.shape[0] != self.batch_size: raise RuntimeError( "Expect input_ids.shape[0] to be LogitsProcessor.batch_size." + f"Got {input_ids.shape[0]} for the former, and {self.batch_size} for the latter." ) if not self.prefilled: # Have not sampled a token yet self.prefilled = True else: for i in range(self.batch_size): if not self.matchers[i].is_terminated(): sampled_token = input_ids[i][-1].item() assert self.matchers[i].accept_token(sampled_token) for i in range(self.batch_size): if not self.matchers[i].is_terminated(): self.matchers[i].fill_next_token_bitmask(self.token_bitmask, i) # We only support masking logits on CUDA or CPU device_type = scores.device.type if device_type != "cuda": scores = scores.to("cpu") xgr.apply_token_bitmask_inplace(scores, self.token_bitmask.to(scores.device)) if device_type != "cuda": scores = scores.to(device_type) # NOTE: Cannot reset here because __call__ is not invoked when stop token # is sampled. This is why each `generate()` call needs to instantiate an # LogitsProcessor return scores xgrammar-0.2.3/python/xgrammar/contrib/mlxlm.py000066400000000000000000000053741521764210300216460ustar00rootroot00000000000000""" Usage: python mlxlm.py --model mlx-community/Qwen2.5-Coder-32B-Instruct-3bit """ import argparse import mlx.core as mx from mlx_lm.generate import generate as mlx_generate from mlx_lm.utils import load as mlx_load from transformers import AutoTokenizer import xgrammar from xgrammar.kernels import apply_token_bitmask_inplace_kernels class XGrammarLogitsProcessor: def __init__(self, grammar: xgrammar.CompiledGrammar, max_rollback_tokens: int = 16): self.matcher = xgrammar.GrammarMatcher(grammar, max_rollback_tokens=max_rollback_tokens) self.vocab_size = grammar.tokenizer_info.vocab_size self.bitmask = xgrammar.allocate_token_bitmask(1, self.vocab_size) def __call__(self, tokens: mx.array, logits: mx.array) -> mx.array: assert tokens.size > 0 # In the first call, tokens.size == #tokens in prompt last_token = tokens[-1].item() acc = self.matcher.accept_token(last_token) if not self.matcher.is_terminated() else False if not acc: self.matcher.reset() self.matcher.accept_token(last_token) if not self.matcher.is_terminated(): self.matcher.fill_next_token_bitmask(self.bitmask) return apply_token_bitmask_inplace_kernels["metal"]( mx.array(self.bitmask.numpy()), logits, self.vocab_size ) return logits def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("--model", type=str, required=True) parser.add_argument( "--prompt", type=str, default="Generate a simple example JSON. No text. Only the JSON" ) parser.add_argument("--seed", type=int, default=42) return parser.parse_args() def main(): args = parse_args() model, _ = mlx_load(args.model) tokenizer = AutoTokenizer.from_pretrained(args.model) mx.random.seed(args.seed) with_logits_processor = mlx_generate( model=model, tokenizer=tokenizer, prompt=tokenizer.apply_chat_template( [{"role": "user", "content": args.prompt}], add_generation_prompt=True ), verbose=False, logits_processors=[ XGrammarLogitsProcessor( grammar=xgrammar.GrammarCompiler( tokenizer_info=xgrammar.TokenizerInfo.from_huggingface(tokenizer) ).compile_builtin_json_grammar() ) ], ) without_logits_processor = mlx_generate( model=model, tokenizer=tokenizer, prompt=tokenizer.apply_chat_template( [{"role": "user", "content": args.prompt}], add_generation_prompt=True ), verbose=False, ) assert without_logits_processor == with_logits_processor print(without_logits_processor) if __name__ == "__main__": main() xgrammar-0.2.3/python/xgrammar/exception.py000066400000000000000000000013221521764210300210400ustar00rootroot00000000000000"""Exceptions in XGrammar.""" from tvm_ffi import register_error class DeserializeFormatError(RuntimeError): """Raised when the deserialization format is invalid.""" class DeserializeVersionError(RuntimeError): """Raised when the serialization format is invalid.""" class InvalidStructuralTagError(RuntimeError): """Raised when the structural tag is invalid.""" class InvalidJSONError(RuntimeError): """Raised when the JSON is invalid.""" register_error("DeserializeFormatError", DeserializeFormatError) register_error("DeserializeVersionError", DeserializeVersionError) register_error("InvalidStructuralTagError", InvalidStructuralTagError) register_error("InvalidJSONError", InvalidJSONError) xgrammar-0.2.3/python/xgrammar/grammar.py000066400000000000000000000404471521764210300205030ustar00rootroot00000000000000"""This module provides classes representing grammars.""" import json from typing import Any, Dict, List, Optional, Tuple, Type, Union, overload from pydantic import BaseModel from typing_extensions import deprecated from .base import XGRObject, _core from .structural_tag import StructuralTag, StructuralTagItem def _convert_instance_to_str(instance: Union[str, Dict[str, Any], StructuralTag]) -> str: """Convert a instance to a string representation. It returns the schema in string format because it's faster to send to C++. This function handles different instance input types and converts them to a JSON string: - StructuralTag. - String inputs are returned as-is (assumed to be valid JSON) - Dictionary inputs are converted to JSON strings Parameters ---------- instance : Union[str, StructuralTag, Dict[str, Any]] The instance to convert, which can be a StructuralTag, a JSON schema string, or a dictionary representing a JSON schema. Returns ------- str The JSON schema as a string. Raises ------ ValueError When the instance type is not supported. TypeError When he dictionary is not serializable. """ if isinstance(instance, dict): return json.dumps(instance) elif isinstance(instance, str): return instance elif isinstance(instance, StructuralTag): return instance.model_dump_json() else: raise ValueError("Invalid instance type") def _convert_schema_to_str(schema: Union[str, Type[BaseModel], Dict[str, Any]]) -> str: """Convert a schema to a string representation. It returns the schema in string format because it's faster to send to C++. This function handles different schema input types and converts them to a JSON string: - Pydantic models are converted using their schema methods - String inputs are returned as-is (assumed to be valid JSON) - Dictionary inputs are converted to JSON strings Parameters ---------- schema : Union[str, Type[BaseModel], Dict[str, Any]] The schema to convert, which can be a Pydantic model class, a JSON schema string, or a dictionary representing a JSON schema. Returns ------- str The JSON schema as a string. Raises ------ ValueError When the schema type is not supported. TypeError When the dictionary is not serializable. """ if isinstance(schema, type) and issubclass(schema, BaseModel): if hasattr(schema, "model_json_schema"): return json.dumps(schema.model_json_schema()) if hasattr(schema, "schema_json"): return json.dumps(schema.schema_json()) else: raise ValueError("The schema should have a model_json_schema or json_schema method.") elif isinstance(schema, str): return schema elif isinstance(schema, dict): return json.dumps(schema) else: raise ValueError("The schema should be a string or a Pydantic model.") def _get_structural_tag_str_from_args(args: List[Any], kwargs: Dict[str, Any]) -> str: """Get the structural tag string from the arguments. It returns the structural tag in string format because it's faster to send to C++. Parameters ---------- args : List[Any] The positional arguments. kwargs : Dict[str, Any] The keyword arguments. Returns ------- str The structural tag string. Raises ------ TypeError When the arguments are invalid. """ if len(args) == 1: if isinstance(args[0], (str, dict, StructuralTag)): return _convert_instance_to_str(args[0]) else: raise TypeError("Invalid argument type for from_structural_tag") elif len(args) == 2 and isinstance(args[0], list) and isinstance(args[1], list): return StructuralTag.from_legacy_structural_tag(args[0], args[1]).model_dump_json( indent=None ) elif "structural_tag" in kwargs: return _convert_instance_to_str(kwargs["structural_tag"]) elif "tags" in kwargs and "triggers" in kwargs: return StructuralTag.from_legacy_structural_tag( kwargs["tags"], kwargs["triggers"] ).model_dump_json(indent=None) else: raise TypeError("Invalid arguments for from_structural_tag") class Grammar(XGRObject): """This class represents a grammar object in XGrammar, and can be used later in the grammar-guided generation. The Grammar object supports context-free grammar (CFG). EBNF (extended Backus-Naur Form) is used as the format of the grammar. There are many specifications for EBNF in the literature, and we follow the specification of GBNF (GGML BNF) in https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md. When printed, the grammar will be converted to GBNF format. """ def __str__(self) -> str: """Print the BNF grammar to a string, in EBNF format. Returns ------- grammar_string : str The BNF grammar string. """ return str(self._handle.to_string()) @staticmethod def from_ebnf(ebnf_string: str, *, root_rule_name: str = "root") -> "Grammar": """Construct a grammar from EBNF string. The EBNF string should follow the format in https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md. Parameters ---------- ebnf_string : str The grammar string in EBNF format. root_rule_name : str, default: "root" The name of the root rule in the grammar. Raises ------ RuntimeError When converting the regex pattern fails, with details about the parsing error. """ return Grammar._create_from_handle(_core.Grammar.from_ebnf(ebnf_string, root_rule_name)) @staticmethod def from_json_schema( schema: Union[str, Type[BaseModel], Dict[str, Any]], *, any_whitespace: bool = True, indent: Optional[int] = None, separators: Optional[Tuple[str, str]] = None, strict_mode: bool = True, max_whitespace_cnt: Optional[int] = None, print_converted_ebnf: bool = False, any_order: bool = False, ) -> "Grammar": """Construct a grammar from JSON schema. Pydantic model or JSON schema string can be used to specify the schema. It allows any whitespace by default. If user want to specify the format of the JSON, set `any_whitespace` to False and use the `indent` and `separators` parameters. The meaning and the default values of the parameters follows the convention in json.dumps(). It internally converts the JSON schema to a EBNF grammar. Parameters ---------- schema : Union[str, Type[BaseModel], Dict[str, Any]] The schema string or Pydantic model or JSON schema dict. any_whitespace : bool, default: True Whether to use any whitespace. If True, the generated grammar will ignore the indent and separators parameters, and allow any whitespace. indent : Optional[int], default: None The number of spaces for indentation. If None, the output will be in one line. Note that specifying the indentation means forcing the LLM to generate JSON strings strictly formatted. However, some models may tend to generate JSON strings that are not strictly formatted. In this case, forcing the LLM to generate strictly formatted JSON strings may degrade the generation quality. See for more details. separators : Optional[Tuple[str, str]], default: None Two separators used in the schema: comma and colon. Examples: (",", ":"), (", ", ": "). If None, the default separators will be used: (",", ": ") when the indent is not None, and (", ", ": ") otherwise. strict_mode : bool, default: True Whether to use strict mode. In strict mode, the generated grammar will not allow properties and items that is not specified in the schema. This is equivalent to setting unevaluatedProperties and unevaluatedItems to false. This helps LLM to generate accurate output in the grammar-guided generation with JSON schema. max_whitespace_cnt : Optional[int], default: None The maximum number of whitespace characters allowed between elements, such like keys, values, separators and so on. If None, there is no limit on the number of whitespace characters. If specified, it will limit the number of whitespace characters to at most max_whitespace_cnt. It should be a positive integer. print_converted_ebnf : bool, default: False If True, the converted EBNF string will be printed. For debugging purposes. any_order : bool, default: False Whether object properties may appear in any order. - False: properties follow the schema's declared order, fully validated (required keys present, no duplicates). - True: properties may appear in any order; only key validity and each key's value schema are enforced. Key presence and uniqueness are not checked, so required keys may be missing and keys may repeat. The entry count is bounded to ``[max(minProperties, n_required), maxProperties]`` (unbounded when maxProperties is unset). Applies to every object, nested included. Returns ------- grammar : Grammar The constructed grammar. Raises ------ RuntimeError When converting the json schema fails, with details about the parsing error. """ schema_str = _convert_schema_to_str(schema) return Grammar._create_from_handle( _core.Grammar.from_json_schema( schema_str, any_whitespace, indent, separators, strict_mode, max_whitespace_cnt, print_converted_ebnf, any_order, ) ) @staticmethod def from_regex(regex_string: str, *, print_converted_ebnf: bool = False) -> "Grammar": """Create a grammar from a regular expression string. Parameters ---------- regex_string : str The regular expression pattern to create the grammar from. print_converted_ebnf : bool, default: False This method will convert the regex pattern to EBNF first. If this is true, the converted EBNF string will be printed. For debugging purposes. Default: False. Returns ------- grammar : Grammar The constructed grammar from the regex pattern. Raises ------ RuntimeError When parsing the regex pattern fails, with details about the parsing error. """ return Grammar._create_from_handle( _core.Grammar.from_regex(regex_string, print_converted_ebnf) ) @overload @staticmethod def from_structural_tag( structural_tag: Union[StructuralTag, str, Dict[str, Any]] ) -> "Grammar": ... @overload @staticmethod @deprecated( "from_structural_tag(tags, triggers) is deprecated. Construct structural tag with the " "StructuralTag class instead." ) def from_structural_tag(tags: List[StructuralTagItem], triggers: List[str]) -> "Grammar": ... @staticmethod def from_structural_tag(*args, **kwargs) -> "Grammar": """Create a grammar from a structural tag. See the Structural Tag Usage in XGrammar documentation for its usage. This method supports two calling patterns: 1. Single structural tag parameter: from_structural_tag(structural_tag) 2. Legacy pattern (deprecated): from_structural_tag(tags, triggers) Parameters ---------- structural_tag : Union[StructuralTag, str, Dict[str, Any]] The structural tag either as a StructuralTag object, or a JSON string or a dictionary. tags : List[StructuralTagItem] (Deprecated) The structural tags. Use StructuralTag class instead. triggers : List[str] (Deprecated) The triggers. Use StructuralTag class instead. Returns ------- grammar : Grammar The constructed grammar from the structural tag. Raises ------ InvalidJSONError When the structural tag is not a valid JSON string. InvalidStructuralTagError When the structural tag is not valid. TypeError When the arguments are invalid. Notes ----- The legacy pattern from_structural_tag(tags, triggers) is deprecated. Use the StructuralTag class to construct structural tags instead. For the deprecated pattern: The structural tag handles the dispatching of different grammars based on the tags and triggers: it initially allows any output, until a trigger is encountered, then dispatch to the corresponding tag; when the end tag is encountered, the grammar will allow any following output, until the next trigger is encountered. See the Advanced Topics of the Structural Tag in XGrammar documentation for its semantic. Structural Tag in XGrammar documentation for its semantic. """ structural_tag_str = _get_structural_tag_str_from_args(args, kwargs) return Grammar._create_from_handle(_core.Grammar.from_structural_tag(structural_tag_str)) @staticmethod def builtin_json_grammar() -> "Grammar": """Get the grammar of standard JSON. This is compatible with the official JSON grammar specification in https://www.json.org/json-en.html. Returns ------- grammar : Grammar The JSON grammar. """ return Grammar._create_from_handle(_core.Grammar.builtin_json_grammar()) @staticmethod def concat(*grammars: "Grammar") -> "Grammar": """Create a grammar that matches the concatenation of the grammars in the list. That is equivalent to using the `+` operator to concatenate the grammars in the list. Parameters ---------- grammars : List[Grammar] The grammars to create the concatenation of. Returns ------- grammar : Grammar The concatenation of the grammars. """ grammar_handles = [grammar._handle for grammar in grammars] return Grammar._create_from_handle(_core.Grammar.concat(grammar_handles)) @staticmethod def union(*grammars: "Grammar") -> "Grammar": """Create a grammar that matches any of the grammars in the list. That is equivalent to using the `|` operator to concatenate the grammars in the list. Parameters ---------- grammars : List[Grammar] The grammars to create the union of. Returns ------- grammar : Grammar The union of the grammars. """ grammar_handles = [grammar._handle for grammar in grammars] return Grammar._create_from_handle(_core.Grammar.union(grammar_handles)) def serialize_json(self) -> str: """Serialize the grammar to a JSON string. Returns ------- json_string : str The JSON string. """ return str(self._handle.serialize_json()) @staticmethod def deserialize_json(json_string: str) -> "Grammar": """Deserialize a grammar from a JSON string. Parameters ---------- json_string : str The JSON string. Returns ------- grammar : Grammar The deserialized grammar. Raises ------ InvalidJSONError When the JSON string is invalid. DeserializeFormatError When the JSON string does not follow the serialization format of the grammar. DeserializeVersionError When the __VERSION__ field in the JSON string is not the same as the current version. """ return Grammar._create_from_handle(_core.Grammar.deserialize_json(json_string)) xgrammar-0.2.3/python/xgrammar/kernels/000077500000000000000000000000001521764210300201355ustar00rootroot00000000000000xgrammar-0.2.3/python/xgrammar/kernels/__init__.py000066400000000000000000000004031521764210300222430ustar00rootroot00000000000000"""The kernels for XGrammar. There are 5 implementations: - CPU: used for CPU tensors - CUDA: not used in the current implementation - Triton: used for CUDA GPU tensors - MLX: used for MLX tensors - Torch Compile: used for torch tensors on other devices """ xgrammar-0.2.3/python/xgrammar/kernels/apply_token_bitmask_inplace_cpu.py000066400000000000000000000055721521764210300271210ustar00rootroot00000000000000"""CPU implementation for in-place applying token mask.""" from typing import List, Optional, Union import torch from ..base import _core def apply_token_bitmask_inplace_cpu( logits: torch.Tensor, bitmask: torch.Tensor, vocab_size: Optional[int] = None, indices: Optional[Union[List[int], torch.Tensor]] = None, ) -> None: """Apply token bitmask in-place on CPU.""" if logits.device.type != "cpu": raise ValueError("logits must be on CPU") if bitmask.device.type != "cpu": raise ValueError("bitmask must be on CPU") if bitmask.dtype != torch.int32: raise ValueError("bitmask must be of type int32") if logits.dim() != 1 and logits.dim() != 2: raise ValueError("logits should be 1D or 2D, but got {}D".format(logits.dim())) if bitmask.dim() != 1 and bitmask.dim() != 2: raise ValueError("bitmask should be 1D or 2D, but got {}D".format(bitmask.dim())) logits_shape = (1, logits.shape[0]) if logits.dim() == 1 else (logits.shape[0], logits.shape[1]) logits_stride = logits.stride() logits_stride = ( (logits_stride[0], 1) if logits.dim() == 1 else (logits_stride[0], logits_stride[1]) ) bitmask_shape = ( (1, bitmask.shape[0]) if bitmask.dim() == 1 else (bitmask.shape[0], bitmask.shape[1]) ) bitmask_stride = bitmask.stride() bitmask_stride = ( (bitmask_stride[0], 1) if bitmask.dim() == 1 else (bitmask_stride[0], bitmask_stride[1]) ) vocab_size = min(logits.shape[-1], bitmask.shape[-1] * 32) if vocab_size is None else vocab_size indices_list = None if indices is not None: if isinstance(indices, torch.Tensor): indices_list = indices.tolist() elif isinstance(indices, list): indices_list = indices if logits.dtype == torch.float32: _core.kernels.apply_token_bitmask_inplace_cpu( logits.data_ptr(), logits_shape, logits_stride, bitmask.data_ptr(), bitmask_shape, bitmask_stride, vocab_size, indices_list, "float32", ) elif logits.dtype == torch.bfloat16: _core.kernels.apply_token_bitmask_inplace_cpu( logits.data_ptr(), logits_shape, logits_stride, bitmask.data_ptr(), bitmask_shape, bitmask_stride, vocab_size, indices_list, "bfloat16", ) elif logits.dtype == torch.float16: _core.kernels.apply_token_bitmask_inplace_cpu( logits.data_ptr(), logits_shape, logits_stride, bitmask.data_ptr(), bitmask_shape, bitmask_stride, vocab_size, indices_list, "float16", ) else: raise ValueError("logits must be of type float32 or bfloat16/float16") xgrammar-0.2.3/python/xgrammar/kernels/apply_token_bitmask_inplace_cuda.cu000066400000000000000000000234731521764210300272250ustar00rootroot00000000000000/* * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // clang-format off #include #include #include #include #include // clang-format on #ifndef CUDART_INF_FP16 #define CUDART_INF_FP16 __ushort_as_half((unsigned short)0x7C00U) #endif #ifndef CUDART_INF_BF16 #define CUDART_INF_BF16 __ushort_as_bfloat16((unsigned short)0x7F80U) #endif constexpr int32_t BITS_PER_BLOCK = 32; constexpr int32_t THREADS_PER_THREAD_BLOCK = 256; template __device__ T NegativeInfinity() { return -INFINITY; } template <> __device__ __half NegativeInfinity<__half>() { return -CUDART_INF_FP16; } template <> __device__ __nv_bfloat16 NegativeInfinity<__nv_bfloat16>() { return -CUDART_INF_BF16; } template __device__ PackedT PackedNegativeInfinity() { constexpr int kAlignment = sizeof(PackedT) / sizeof(T); T packed[kAlignment]; #pragma unroll for (int i = 0; i < kAlignment; i++) { packed[i] = NegativeInfinity(); } return *reinterpret_cast(packed); } template __global__ void __launch_bounds__(THREADS_PER_THREAD_BLOCK) LogitsBitmaskKernel( T* __restrict__ logits, const int32_t* __restrict__ bitmask, const int32_t* __restrict__ indices, int32_t vocab_size, int32_t logits_stride, int32_t bitmask_stride ) { constexpr int kAlignment = sizeof(PackedT) / sizeof(T); constexpr uint32_t kPackedMask = (1 << kAlignment) - 1; const int batch_idx = (indices == nullptr) ? blockIdx.y : indices[blockIdx.y]; const int block_offset = blockIdx.x * THREADS_PER_THREAD_BLOCK * kBitsPerThread; T* logits_gmem_ptr = logits + batch_idx * logits_stride + block_offset; const int32_t* bitmask_gmem_ptr = bitmask + batch_idx * bitmask_stride + block_offset / BITS_PER_BLOCK; const int bitmask_inner_idx = threadIdx.x % (BITS_PER_BLOCK / kAlignment); T logits_reg[kAlignment]; #pragma unroll for (int offset = threadIdx.x * kAlignment; offset < THREADS_PER_THREAD_BLOCK * kBitsPerThread; offset += THREADS_PER_THREAD_BLOCK * kAlignment) { if (block_offset + offset >= vocab_size) { break; } const uint32_t bitmask_val = (~bitmask_gmem_ptr[offset / BITS_PER_BLOCK] >> (bitmask_inner_idx * kAlignment)) & kPackedMask; if (bitmask_val == 0) { continue; } if (bitmask_val == kPackedMask) { *reinterpret_cast(logits_gmem_ptr + offset) = PackedNegativeInfinity(); continue; } *reinterpret_cast(logits_reg) = *reinterpret_cast(logits_gmem_ptr + offset); #pragma unroll for (int i = 0; i < kAlignment; i++) { if (((bitmask_val >> i) & 1)) { logits_reg[i] = NegativeInfinity(); } } *reinterpret_cast(logits_gmem_ptr + offset) = *reinterpret_cast(logits_reg); } } template ::value>> constexpr auto CeilDiv(T numerator, T denominator) { return (numerator + denominator - 1) / denominator; } template void ApplyTokenBitmaskInplaceDispatchToBitsPerThread( T* __restrict__ logits, const int32_t* __restrict__ bitmask, const int32_t* __restrict__ indices, int32_t vocab_size, int32_t logits_stride, int32_t bitmask_stride, int32_t num_rows ) { constexpr int kAlignment = sizeof(PackedT) / sizeof(T); const int32_t num_blocks_per_row = CeilDiv(2048 / THREADS_PER_THREAD_BLOCK * 128, num_rows); const int32_t num_bits_per_thread = CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * num_blocks_per_row); const dim3 block(THREADS_PER_THREAD_BLOCK); cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); if (num_bits_per_thread <= 4 && kAlignment <= 4) { const dim3 grid(CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * 4), num_rows); LogitsBitmaskKernel<<>>( logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride ); } else if (num_bits_per_thread <= 8 && kAlignment <= 8) { const dim3 grid(CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * 8), num_rows); LogitsBitmaskKernel<<>>( logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride ); } else if (num_bits_per_thread <= 16 && kAlignment <= 16) { const dim3 grid(CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * 16), num_rows); LogitsBitmaskKernel<<>>( logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride ); } else { const dim3 grid(CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * 32), num_rows); LogitsBitmaskKernel<<>>( logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride ); } } template void ApplyTokenBitmaskInplaceDispatchToPackedT( T* __restrict__ logits, const int32_t* __restrict__ bitmask, const int32_t* __restrict__ indices, int32_t vocab_size, int32_t logits_stride, int32_t bitmask_stride, int32_t num_rows ) { if (logits_stride % (sizeof(float4) / sizeof(T)) == 0) { ApplyTokenBitmaskInplaceDispatchToBitsPerThread( logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride, num_rows ); } else { ApplyTokenBitmaskInplaceDispatchToBitsPerThread( logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride, num_rows ); } } void ApplyTokenBitmaskInplace( at::Tensor logits, at::Tensor bitmask, at::optional indices = at::nullopt ) { TORCH_CHECK(logits.is_cuda(), "logits must be a CUDA tensor."); TORCH_CHECK(logits.dim() == 1 || logits.dim() == 2, "logits must be a 1D or 2D tensor."); std::pair logits_shape = logits.dim() == 2 ? std::make_pair( static_cast(logits.size(0)), static_cast(logits.size(1)) ) : std::make_pair(1, static_cast(logits.size(0))); TORCH_CHECK(bitmask.is_cuda(), "bitmask must be a CUDA tensor."); TORCH_CHECK(bitmask.dim() == 1 || bitmask.dim() == 2, "bitmask must be a 1D or 2D tensor."); std::pair bitmask_shape = bitmask.dim() == 2 ? std::make_pair( static_cast(bitmask.size(0)), static_cast(bitmask.size(1)) ) : std::make_pair(1, static_cast(bitmask.size(0))); TORCH_CHECK(bitmask.dtype() == torch::kInt32, "bitmask must be of type int32."); TORCH_CHECK( (logits_shape.second + BITS_PER_BLOCK - 1) / BITS_PER_BLOCK >= bitmask_shape.second, "The provided logits's vocab size should be no less than the bitmask's vocab size " "(converted from bitmask size). But got vocab size ", logits_shape.second, " vs bitmask size ", bitmask_shape.second ); TORCH_CHECK(logits.dim() == 1 or logits.stride(1) == 1, "logits's stride(1) must be 1"); TORCH_CHECK(bitmask.dim() == 1 or bitmask.stride(1) == 1, "bitmask's stride(1) must be 1"); int vocab_size = std::min(logits_shape.second, bitmask_shape.second * BITS_PER_BLOCK); int32_t num_rows = logits_shape.first; int32_t* indices_ptr = nullptr; if (indices) { TORCH_CHECK(indices->is_cuda(), "indices must be a CUDA tensor."); TORCH_CHECK(indices->is_contiguous(), "indices must be contiguous."); TORCH_CHECK(indices->dim() == 1, "indices must be a 1D tensor."); TORCH_CHECK(indices->dtype() == torch::kInt32, "indices must be of type int32."); num_rows = indices->size(0); indices_ptr = indices->data_ptr(); } else { TORCH_CHECK( logits_shape.first == bitmask_shape.first, "logits and bitmask must have the same batch size." ); } switch (logits.scalar_type()) { case torch::kFloat32: { ApplyTokenBitmaskInplaceDispatchToPackedT( logits.data_ptr(), bitmask.data_ptr(), indices_ptr, vocab_size, logits.stride(0), bitmask.stride(0), num_rows ); break; } case torch::kFloat16: { ApplyTokenBitmaskInplaceDispatchToPackedT( reinterpret_cast<__half*>(logits.data_ptr()), bitmask.data_ptr(), indices_ptr, vocab_size, logits.stride(0), bitmask.stride(0), num_rows ); break; } case torch::kBFloat16: { ApplyTokenBitmaskInplaceDispatchToPackedT( reinterpret_cast<__nv_bfloat16*>(logits.data_ptr()), bitmask.data_ptr(), indices_ptr, vocab_size, logits.stride(0), bitmask.stride(0), num_rows ); break; } default: TORCH_CHECK(false, "logits dtype must be float, half or bfloat16."); break; } } TORCH_LIBRARY_FRAGMENT(TORCH_EXTENSION_NAME, m) { m.def( "apply_token_bitmask_inplace_cuda(Tensor logits, Tensor bitmask, Tensor? indices=None) -> ()" ); } TORCH_LIBRARY_IMPL(TORCH_EXTENSION_NAME, CUDA, m) { m.impl("apply_token_bitmask_inplace_cuda", &ApplyTokenBitmaskInplace); } xgrammar-0.2.3/python/xgrammar/kernels/apply_token_bitmask_inplace_cuda.py000066400000000000000000000100361521764210300272350ustar00rootroot00000000000000# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import platform from contextlib import suppress from typing import List, Optional, Union import torch import torch.utils.cpp_extension def _check_cuda_toolchain() -> None: """check if nvcc is available and if pytorch will likely find it""" import glob import os import shutil from pathlib import Path # First check if CUDA is available in PyTorch if not torch.cuda.is_available(): raise ImportError("CUDA is not available in PyTorch") # This is similar logic to what pytorch does to find the nvcc compiler nvcc_path = shutil.which("nvcc") if nvcc_path is None: cuda_home = os.environ.get("CUDA_HOME", os.environ.get("CUDA_PATH", None)) if cuda_home is None: if os.name == "nt": # This is a very hardcoded asumption about install directories but pytorch does this. cuda_homes = glob.glob("C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v*.*") if len(cuda_homes) == 0: cuda_home = "" else: cuda_home = cuda_homes[0] else: cuda_home = "/usr/local/cuda" if cuda_home is None: raise ImportError("No CUDA toolchain found") nvcc_path = str(Path(cuda_home) / "bin" / "nvcc") if not os.path.exists(nvcc_path): raise ImportError(f"nvcc compiler not found at {nvcc_path}") def _remove_torch_nvcc_flags() -> None: REMOVE_NVCC_FLAGS = [ "-D__CUDA_NO_HALF_OPERATORS__", "-D__CUDA_NO_HALF_CONVERSIONS__", "-D__CUDA_NO_BFLOAT16_CONVERSIONS__", "-D__CUDA_NO_HALF2_OPERATORS__", ] for flag in REMOVE_NVCC_FLAGS: with suppress(ValueError): torch.utils.cpp_extension.COMMON_NVCC_FLAGS.remove(flag) def _load_torch_ops() -> None: from pathlib import Path torch_op_file_path = Path(__file__).with_suffix(".cu") with open(torch_op_file_path) as f: source = f.read() cflags = ["-O3"] if platform.system() != "Windows": cflags.append("-Wno-switch-bool") cuda_cflags = ["-O3", "-std=c++17", "--threads", "4", "-use_fast_math"] # Use the safer cpp_extension.load_inline instead of cpp_extension.load torch.utils.cpp_extension.load_inline( name="xgrammar", cpp_sources=[], # No C++ sources cuda_sources=[source], extra_cflags=cflags, extra_cuda_cflags=cuda_cflags, with_cuda=True, is_python_module=False, ) _check_cuda_toolchain() _remove_torch_nvcc_flags() _load_torch_ops() _is_register_fake_available = hasattr(torch, "library") and hasattr(torch.library, "register_fake") if _is_register_fake_available: # To support torch.compile with fullgraph=True, a fake kernel is needed. @torch.library.register_fake("xgrammar::apply_token_bitmask_inplace_cuda") def _( logits: torch.Tensor, bitmask: torch.Tensor, indices: Optional[torch.Tensor] = None ) -> None: pass def apply_token_bitmask_inplace_cuda( logits: torch.Tensor, bitmask: torch.Tensor, indices: Optional[Union[List[int], torch.Tensor]] = None, ) -> None: if isinstance(indices, list): indices = torch.tensor(indices, dtype=torch.int32, device=logits.device) if indices is not None: indices = indices.to(logits.device) torch.ops.xgrammar.apply_token_bitmask_inplace_cuda(logits, bitmask, indices) xgrammar-0.2.3/python/xgrammar/kernels/apply_token_bitmask_inplace_torch.py000066400000000000000000000037421521764210300274460ustar00rootroot00000000000000from typing import List, Optional import torch def apply_token_bitmask_inplace_kernel_no_indices_torch( logits: torch.Tensor, bitmask: torch.Tensor, vocab_size: int ) -> None: # logits: (batch_size, vocab_size) # bitmask: (batch_size, bitmask_size) # mask_expanded: (batch_size, 32 * bitmask_size) mask_expanded = torch.repeat_interleave(bitmask, 32, dim=-1) # bit_indices: (32 * bitmask_size,) bit_indices = torch.arange(32, device=logits.device, dtype=torch.int32).repeat( bitmask.shape[-1] ) # bit_masks: (batch_size, 32 * bitmask_size) bit_masks = (mask_expanded >> bit_indices) & 1 bit_masks = bit_masks[..., :vocab_size] logits[..., :vocab_size] = logits[..., :vocab_size].masked_fill_(bit_masks == 0, float("-inf")) def apply_token_bitmask_inplace_kernel_indices_torch( logits: torch.Tensor, bitmask: torch.Tensor, vocab_size: int, indices: List[int] ) -> None: # logits: (batch_size, vocab_size) # bitmask: (batch_size, bitmask_size) # mask_expanded: (batch_size, 32 * bitmask_size) mask_expanded = torch.repeat_interleave(bitmask[indices], 32, dim=-1) # bit_indices: (32 * bitmask_size,) bit_indices = torch.arange(32, device=logits.device, dtype=torch.int32).repeat( bitmask.shape[-1] ) bit_masks = (mask_expanded >> bit_indices) & 1 bit_masks = bit_masks[..., :vocab_size] logits[indices, :vocab_size] = logits[indices, :vocab_size].masked_fill_( bit_masks == 0, float("-inf") ) def apply_token_bitmask_inplace_torch( logits: torch.Tensor, bitmask: torch.Tensor, vocab_size: Optional[int] = None, indices: Optional[List[int]] = None, ) -> None: vocab_size = min(logits.shape[-1], bitmask.shape[-1] * 32) if vocab_size is None else vocab_size if indices is None: apply_token_bitmask_inplace_kernel_no_indices_torch(logits, bitmask, vocab_size) else: apply_token_bitmask_inplace_kernel_indices_torch(logits, bitmask, vocab_size, indices) xgrammar-0.2.3/python/xgrammar/kernels/apply_token_bitmask_inplace_torch_compile.py000066400000000000000000000041321521764210300311500ustar00rootroot00000000000000from typing import List, Optional import torch @torch.compile(dynamic=True) def apply_token_bitmask_inplace_kernel_no_indices_torch_compile( logits: torch.Tensor, bitmask: torch.Tensor, vocab_size: int ) -> None: # logits: (batch_size, vocab_size) # bitmask: (batch_size, bitmask_size) # mask_expanded: (batch_size, 32 * bitmask_size) mask_expanded = torch.repeat_interleave(bitmask, 32, dim=-1) # bit_indices: (32 * bitmask_size,) bit_indices = torch.arange(32, device=logits.device, dtype=torch.int32).repeat( bitmask.shape[-1] ) # bit_masks: (batch_size, 32 * bitmask_size) bit_masks = (mask_expanded >> bit_indices) & 1 bit_masks = bit_masks[..., :vocab_size] logits[..., :vocab_size] = logits[..., :vocab_size].masked_fill_(bit_masks == 0, float("-inf")) @torch.compile(dynamic=True) def apply_token_bitmask_inplace_kernel_indices_torch_compile( logits: torch.Tensor, bitmask: torch.Tensor, vocab_size: int, indices: List[int] ) -> None: # logits: (batch_size, vocab_size) # bitmask: (batch_size, bitmask_size) # mask_expanded: (batch_size, 32 * bitmask_size) mask_expanded = torch.repeat_interleave(bitmask[indices], 32, dim=-1) # bit_indices: (32 * bitmask_size,) bit_indices = torch.arange(32, device=logits.device, dtype=torch.int32).repeat( bitmask.shape[-1] ) bit_masks = (mask_expanded >> bit_indices) & 1 bit_masks = bit_masks[..., :vocab_size] logits[indices, :vocab_size] = logits[indices, :vocab_size].masked_fill_( bit_masks == 0, float("-inf") ) def apply_token_bitmask_inplace_torch_compile( logits: torch.Tensor, bitmask: torch.Tensor, vocab_size: Optional[int] = None, indices: Optional[List[int]] = None, ) -> None: vocab_size = min(logits.shape[-1], bitmask.shape[-1] * 32) if vocab_size is None else vocab_size if indices is None: apply_token_bitmask_inplace_kernel_no_indices_torch_compile(logits, bitmask, vocab_size) else: apply_token_bitmask_inplace_kernel_indices_torch_compile( logits, bitmask, vocab_size, indices ) xgrammar-0.2.3/python/xgrammar/kernels/apply_token_bitmask_inplace_triton.py000066400000000000000000000102651521764210300276440ustar00rootroot00000000000000from typing import List, Optional, Union import torch try: import triton import triton.language as tl except ImportError as err: raise ImportError("Triton is not installed") from err @triton.jit def apply_token_bitmask_inplace_kernel( logits_ptr, bitmask_ptr, indices_ptr, num_rows, vocab_size, logits_strides, bitmask_strides, NUM_SMS: tl.constexpr, BLOCK_SIZE: tl.constexpr, ): """Apply a bitmask to logits in-place using Triton. The bitmask is a 01 bitwise compressed tensor, where 0 means the token is masked and 1 means the token is not masked. After applying the bitmask, the masked logits will be set to -inf. Parameters ---------- logits_ptr : tl.tensor Pointer to the logits tensor to apply the bitmask to. bitmask_ptr : tl.tensor Pointer to the bitmask tensor to apply. indices_ptr : Optional[tl.tensor] Optional pointer to indices tensor specifying which rows to apply the mask to. num_rows : int Number of rows to process. If indices_ptr is provided, this is the number of unique indices. vocab_size : int Size of the vocabulary dimension. If the logits does not have a vocab padding, this is the same as the logits's second dimension. Otherwise, this is the actual size of the vocabulary. logits_strides : int Stride between rows in the logits tensor. bitmask_strides : int Stride between rows in the bitmask tensor. NUM_SMS : int Number of streaming multiprocessors to use. BLOCK_SIZE : int Size of processing blocks. """ pid = tl.program_id(0) num_blocks = tl.cdiv(vocab_size, BLOCK_SIZE) for work_id in tl.range(pid, num_rows * num_blocks, NUM_SMS): row_id = work_id // num_blocks block_offset = (work_id % num_blocks) * BLOCK_SIZE batch_id = row_id if indices_ptr is None else tl.load(indices_ptr + row_id) offsets = block_offset + tl.arange(0, BLOCK_SIZE) bitmask_offsets = block_offset // 32 + tl.arange(0, BLOCK_SIZE // 32) vocab_mask = offsets < vocab_size packed_bitmask_mask = bitmask_offsets < bitmask_strides packed_bitmask = tl.load( bitmask_ptr + batch_id * bitmask_strides + bitmask_offsets, packed_bitmask_mask ) bitmask = ((packed_bitmask[:, None] >> (tl.arange(0, 32)[None, :])) & 1) == 0 bitmask = bitmask.reshape(BLOCK_SIZE) tl.store( logits_ptr + batch_id * logits_strides + offsets, -float("inf"), vocab_mask & bitmask ) def apply_token_bitmask_inplace_triton( logits: torch.Tensor, bitmask: torch.Tensor, vocab_size: Optional[int] = None, indices: Optional[Union[List[int], torch.Tensor]] = None, ): NUM_SMS = torch.cuda.get_device_properties("cuda").multi_processor_count BLOCK_SIZE = 4096 arch = torch.cuda.get_device_properties(0).gcnArchName if torch.version.hip is not None and "gfx1" not in arch: # For AMD GPUs (non-Navi) WARP_SIZE = 64 else: WARP_SIZE = 32 assert bitmask.dtype == torch.int32, "bitmask must be of type int32" detected_vocab_size = min(logits.shape[-1], bitmask.shape[-1] * 32) if vocab_size is None: vocab_size = detected_vocab_size else: assert ( vocab_size <= detected_vocab_size ), f"vocab_size {vocab_size} is larger than the detected vocab_size {detected_vocab_size}" num_rows = len(indices) if indices is not None else logits.shape[0] if logits.ndim == 2 else 1 if indices is not None: if isinstance(indices, torch.Tensor): indices = indices.to(dtype=torch.int32, device=logits.device, non_blocking=True) else: indices_cpu = torch.tensor(indices, dtype=torch.int32) indices = indices_cpu.to(device=logits.device, non_blocking=True) grid = (NUM_SMS,) apply_token_bitmask_inplace_kernel[grid]( logits, bitmask, indices, num_rows, vocab_size, logits.stride()[0], bitmask.stride()[0], NUM_SMS, BLOCK_SIZE, num_warps=BLOCK_SIZE // WARP_SIZE // (16 // logits.element_size()), num_stages=3, ) xgrammar-0.2.3/python/xgrammar/kernels/apply_token_bitmask_mlx.py000066400000000000000000000015631521764210300254330ustar00rootroot00000000000000"""MLX kernel for applying token bitmasks.""" import itertools import mlx.core as mx @mx.compile def apply_token_bitmask_mlx(bitmask: mx.array, logits: mx.array, vocab_size: int): """Apply a token bitmask to logits using MLX for Metal GPUs. Args: bitmask: A tensor of shape (batch_size, (vocab_size + 31) // 32) containing the bitmask. Each bit in the bitmask determines whether the corresponding token is allowed (1) or not (0). logits: A tensor of shape (batch_size, vocab_size) containing the logits. Returns: The logits with -inf for tokens that are not allowed. """ bitmap = mx.array( [l[::-1] for l in itertools.product(*[[float("-inf"), 0]] * 8)], dtype=logits.dtype ) bitmask = bitmask.view(mx.uint8) return logits[..., :vocab_size] + bitmap[bitmask].flatten(-2)[..., :vocab_size] xgrammar-0.2.3/python/xgrammar/load_binding.py000066400000000000000000000004401521764210300214530ustar00rootroot00000000000000"""Load the xgrammar bindings.""" import os from tvm_ffi.libinfo import load_lib_module if os.environ.get("XGRAMMAR_BUILD_DOCS") == "1": # During documentation builds, skip loading the native library. LIB = None else: LIB = load_lib_module("xgrammar", "xgrammar_bindings") xgrammar-0.2.3/python/xgrammar/matcher.py000066400000000000000000000566361521764210300205070ustar00rootroot00000000000000"""Match the output of the LLM to the specified grammar, then generate the mask for the next token. """ import math import warnings from typing import List, Literal, Optional, Tuple, Union import torch from numpy.typing import ArrayLike from .base import XGRObject, _core from .compiler import CompiledGrammar bitmask_dtype = torch.int32 """The dtype of the bitmask: int32.""" def get_bitmask_shape(batch_size: int, vocab_size: int) -> Tuple[int, int]: """Return the shape of the bitmask: (batch_size, ceil(vocab_size / 32)).""" return (batch_size, math.ceil(vocab_size / 32)) _FULL_MASK = torch.tensor(-1, dtype=bitmask_dtype) def allocate_token_bitmask(batch_size: int, vocab_size: int) -> torch.Tensor: """Allocate the bitmask for the next token prediction. The bitmask is an int32 tensor on CPU with shape (batch_size, ceil(vocab_size / 32)). Users who have their own needs to manage CUDA memory can construct the tensor with get_bitmask_shape and bitmask_dtype themselves. The reason why we use int32 instead of uint32 is that old versions of PyTorch do not support uint32. Parameters ---------- batch_size : int The batch size of the bitmask. vocab_size : int The size of the vocabulary. Returns ------- bitmask : torch.Tensor The allocated bitmask. """ # In CUDA, use pinned memory to speed up data transfer from CPU to GPU return torch.full(get_bitmask_shape(batch_size, vocab_size), _FULL_MASK, dtype=bitmask_dtype) def reset_token_bitmask(bitmask: torch.Tensor) -> None: """Reset the bitmask to the full mask.""" bitmask.fill_(_FULL_MASK) def apply_token_bitmask_inplace( logits: torch.Tensor, bitmask: torch.Tensor, *, vocab_size: Optional[int] = None, indices: Optional[List[int]] = None, backend: Literal["auto", "cpu", "cuda", "triton", "torch_compile", "torch_native"] = "auto", ) -> None: """Apply the bitmask to the logits in-place. The bitmask is a 01 bitwise compressed tensor, where 0 means the token is masked and 1 means the token is not masked. It can be generated by allocate_token_bitmask and filled by fill_next_token_bitmask. After applying the bitmask, the masked logits will be set to -inf. The shape of logits and bitmask should be (batch_size, vocab_size) and (batch_size, bitmask_size) respectively. bitmask_size = ceil(vocab_size / 32). The operation is: .. code:: python for i in range(batch_size): for j in range(vocab_size): if get_bitmask_value(bitmask, i, j) == 0: logits[i, j] = -inf get_bitmask_value(bitmask, i, j) gets the j-th bit of the i-th row of the bitmask. Notes ----- Padding: This method allows additional padding on the vocabulary dimension of logits or bitmask. If padding exists, provide the real vocab size to the vocab_size parameter, and the operation will be applied to logits[..., :vocab_size] and bitmask[..., :ceil(vocab_size / 32)]. If vocab_size is not provided, the vocab size will be detected as min(logits.shape[-1], bitmask.shape[-1] * 32). Indices: Indices can be used to specify which logits in the batch to apply the bitmask to. It is especially useful when there are structured requests and unstructured requests mixed in the same batch by skipping masking the logits in the unstructured requests. When specified, the operation will be .. code:: python for batch_id in indices: for j in range(vocab_size): if get_bitmask_value(bitmask, batch_id, j) == 0: logits[batch_id, j] = -inf When indices is specified, the batch sizes of logits and bitmask do not need to be the same. As long as the indices are valid, the operation will be performed. Device: The logits and bitmask should be on the same device. If both them are on GPU, we launch a GPU kernel to apply bitmask. If both them are on CPU, we use a CPU implementation. The GPU kernel is optimized and should be preferred. In practice, the bitmask is allocated on CPU, and the logits is usually on GPU, so users should manually copy the bitmask to GPU before calling this function. Parameters ---------- logits : torch.Tensor The tensor to apply the bitmask to. bitmask : torch.Tensor The bitmask to apply. vocab_size : Optional[int], default: None The size of the vocabulary. If not provided, the vocab size will be detected as min(logits.shape[-1], bitmask.shape[-1] * 32). indices : Optional[List[int]], default: None A list of indices to specify which logits in the batch to apply the bitmask to. Should be unique. If None, apply the bitmask to all logits in the batch. backend : Literal["auto", "cpu", "cuda", "triton", "torch_compile", "torch_native"], default: "auto" The backend where the token bitmask should be applied inplace. If the value is "auto", then it will choose the backend according to the type of the logits. * CPU: CPU implementation * Triton: Default CUDA implementation * Torch Compile: If the hardware is not the above, use this. This supports multiple backends, including CPU/CUDA/TPU/ROCm * CUDA: CUDA native implementation, helpful for C++-based inference engines * Torch Native: Support more backends such as Kunlun. Its performance may be worse than torch compile. """ if bitmask.device != logits.device: raise ValueError( "logits and bitmask should be on the same device. " + f"But got logits.device: {logits.device}, bitmask.device: {bitmask.device}" ) if backend == "auto": if logits.device.type == "cpu": backend = "cpu" elif logits.device.type == "cuda": backend = "triton" else: backend = "torch_compile" # dispatch to different implementations based on the device if backend == "cpu": from .kernels.apply_token_bitmask_inplace_cpu import apply_token_bitmask_inplace_cpu apply_token_bitmask_inplace_cpu(logits, bitmask, vocab_size, indices) elif backend == "torch_native": from .kernels.apply_token_bitmask_inplace_torch import apply_token_bitmask_inplace_torch apply_token_bitmask_inplace_torch(logits, bitmask, vocab_size, indices) elif backend == "triton": from .kernels.apply_token_bitmask_inplace_triton import apply_token_bitmask_inplace_triton apply_token_bitmask_inplace_triton(logits, bitmask, vocab_size, indices) elif backend == "torch_compile": from .kernels.apply_token_bitmask_inplace_torch_compile import ( apply_token_bitmask_inplace_torch_compile, ) apply_token_bitmask_inplace_torch_compile(logits, bitmask, vocab_size, indices) elif backend == "cuda": from .kernels.apply_token_bitmask_inplace_cuda import apply_token_bitmask_inplace_cuda apply_token_bitmask_inplace_cuda(logits, bitmask, indices) else: raise ValueError( f'Unknown backend: {backend}. The value should be one of them: ["auto", "cpu", "cuda", "triton", "torch_compile", "torch_native"]' ) class GrammarMatcher(XGRObject): """Match the output of the LLM to the specified grammar, then generate the mask for the next token. This is the core class in the grammar-guided generation. This class maintains a stateful matcher that can accept tokens and strings, then match them to the specified grammar. The matcher can provide a bitmask for the next token prediction, so that the output of the LLM follows the specified grammar. Its state can be reset and rolled back by tokens. It also provides utilities for jump-forward decoding. After matching the whole grammar, the matcher will accept a stop token. The token mask at this time will only allow stop tokens. After accepting the stop token, the matcher will terminate, then it cannot accept any new token or generate a new token mask, meaning the generation is finished. Under the hood, it utilizes a pushdown automaton with backtracking to match the grammar, with optimizations specific to LLM token mask generation. """ def __init__( self, compiled_grammar: CompiledGrammar, *, override_stop_tokens: Optional[Union[int, List[int]]] = None, terminate_without_stop_token: bool = False, max_rollback_tokens: int = -1, ) -> None: """Construct the grammar matcher. Parameters ---------- compiled_grammar : CompiledGrammar The initialization context for the grammar matcher. override_stop_tokens : Optional[Union[int, List[int]]], default: None If not None, the stop tokens to override the ones in the grammar. terminate_without_stop_token : bool, default: False Whether to terminate the matcher without accepting a stop token. max_rollback_tokens : int, default: -1 Deprecated. You don't need to set it and it's always unlimited (-1). The new Earley parser significantly reduces the number of states, so we can allow unlimited rollback. The maximum number of rollback tokens allowed. The rollback operation is useful for jump-forward decoding and speculative decoding. """ if not isinstance(compiled_grammar, CompiledGrammar): raise ValueError("The grammar should be compiled before passing it to GrammarMatcher.") if not max_rollback_tokens == -1: warnings.warn( "max_rollback_tokens is deprecated. You don't need to set it and it's always " "unlimited (-1).", DeprecationWarning, ) if isinstance(override_stop_tokens, int): override_stop_tokens = [override_stop_tokens] self._init_handle( _core.GrammarMatcher( compiled_grammar._handle, override_stop_tokens, terminate_without_stop_token, max_rollback_tokens, ) ) def accept_token(self, token_id: int, *, debug_print: bool = False) -> bool: """Accept one token and update the state of the matcher. In the following cases, the matcher will not accept the token and return False: 1. The token does not match the grammar. 2. The matcher has terminated after accepting the stop token, but is trying to accept a new token. 3. The token id is out of range. 4. The token is a special token. The user should capture the return value and handle the cases where the token is not accepted. Parameters ---------- token_id : int The id of the token to accept. debug_print : bool, default: False Whether to print information about the internal state of the matcher. Helpful for debugging. Returns ------- accepted : bool Whether the token is accepted. """ return self._handle.accept_token(token_id, debug_print) def accept_string(self, input_str: Union[str, bytes], *, debug_print: bool = False) -> bool: """Accept a string and update the state of the matcher. The whole string is considered as one step in rollback. It is used to complement the functionality of accept_token, and accept_token should always be used to accept tokens. Parameters ---------- input_str : Union[str, bytes] The string to be accepted. debug_print : bool, default: False Whether to print information about the internal state of the matcher. Helpful for debugging. Returns ------- accepted : bool Whether the string is accepted. """ return self._handle.accept_string(input_str, debug_print) def fill_next_token_bitmask( self, bitmask: ArrayLike, index: int = 0, *, debug_print: bool = False ) -> bool: """Fill the bitmask for the next token prediction. The input bitmask can be generated by allocate_token_bitmask, and must be on CPU. bitmask[index] will be filled with the next token bitmask. This method does not change the matcher state. Parameters ---------- bitmask : ArrayLike The bitmask for the next token prediction. It supports torch.Tensor and other array-like objects, as long as they support the DLPack protocol. index : int, default: 0 The batch id of the bitmask. debug_print : bool, default: False Whether to print information about generated bitmask. Helpful for debugging. Returns ------- need_apply : bool Whether the bitmask need to be applied (not all-true). An optimization: if False, this means the bitmask is already all-true, so no need to apply it. Raises ------ RuntimeError If the bitmask is invalid (not on CPU, not int32, shape mismatch). """ return self._handle.fill_next_token_bitmask(bitmask, index, debug_print) def traverse_draft_tree( self, retrieve_next_token: torch.Tensor, retrieve_next_sibling: torch.Tensor, draft_tokens: torch.Tensor, token_bitmask: torch.Tensor, time_threshold: float = -1.0, ) -> bool: """Traverse a draft token tree and fill the token bitmask for each node. Parameters ---------- retrieve_next_token : torch.Tensor 1D int64 tensor where retrieve_next_token[i] gives the index of the child node of node i, or -1 if no child exists. retrieve_next_sibling : torch.Tensor 1D int64 tensor where retrieve_next_sibling[i] gives the index of the sibling node of node i, or -1 if no sibling exists. draft_tokens : torch.Tensor 1D int64 tensor of draft token ids at each node. token_bitmask : torch.Tensor 2D int32 tensor (num_nodes x bitmask_size) to store the generated bitmasks. time_threshold : float Maximum allowed time in seconds for the traversal. If the traversal exceeds this threshold, it returns False. A value <= 0 disables the timeout (default: -1.0). Returns ------- completed : bool True if the traversal completed successfully, False if it timed out. """ return self._handle.traverse_draft_tree( retrieve_next_token, retrieve_next_sibling, draft_tokens, token_bitmask, time_threshold ) def find_jump_forward_string(self) -> str: """Find the jump-forward string for jump-forward decoding. This is the longest string that certainly conforms with the current grammar from the current matcher state. This string can become the output of the LLM without requiring LLM decoding. This method does not change the matcher state. Returns ------- jump_forward_string : str The jump-forward string. """ return str(self._handle.find_jump_forward_string()) def rollback(self, num_tokens: int = 1) -> None: """Rollback the matcher to a previous state by several tokens. Parameters ---------- num_tokens : int, default: 1 The number of tokens to rollback. It cannot exceed the current number of steps, nor can it exceed the specified maximum number of rollback tokens. """ self._handle.rollback(num_tokens) def is_terminated(self) -> bool: """Check if the matcher has terminated. If terminate_without_stop_token is False, the matcher will terminate if it has accepted the stop token. Otherwise, the matcher will terminate after matching the whole grammar. Returns ------- terminated : bool Whether the matcher has terminated. """ return self._handle.is_terminated() def is_completed(self) -> bool: """Check if the input accepted so far forms a complete valid string according to the grammar. Unlike :meth:`is_terminated`, this does not require the stop token to have been accepted. Returns ------- completed : bool Whether the grammar's root rule is fully matched. """ return self._handle.is_completed() def reset(self) -> None: """Reset the matcher to the initial state.""" return self._handle.reset() def fork(self) -> "GrammarMatcher": """Fork the matcher. Returns a new GrammarMatcher sharing the same compiled grammar and tokenizer info, with a deep copy of all other state (parsing state, token history, etc.). Returns ------- forked : GrammarMatcher A new matcher with the same grammar but independent parsing state. """ return GrammarMatcher._create_from_handle(self._handle.fork()) @property def max_rollback_tokens(self) -> int: """Depracated. Now max_rollback_tokens is always unlimited (-1). Get the maximum number of rollback tokens allowed. Returns ------- max_rollback_tokens : int The maximum number of rollback tokens. """ return -1 @property def stop_token_ids(self) -> List[int]: """The ids of the stop tokens used in the matcher. If specified, the provided stop tokens will be used. Otherwise, the stop tokens will be detected from the vocabulary. Returns ------- stop_token_ids : List[int] The ids of the stop tokens. """ return list(self._handle.stop_token_ids()) def _debug_print_internal_state(self) -> str: """Print the internal state of the matcher. This is used for debugging. The representation of the internal state is subject to change. Returns ------- internal_state : str The internal state of the matcher. """ return str(self._handle._debug_print_internal_state()) class BatchGrammarMatcher(XGRObject): """A batch version of GrammarMatcher that can fill the next token bitmask for multiple matchers in parallel. It utilizes multiple threads to speed up the computation. It is especially useful when the batch size is large. """ def __init__(self, max_threads: Union[int, Literal["auto"]] = "auto") -> None: """Construct the batch grammar matcher. Parameters ---------- max_threads : Union[int, Literal["auto"]], default: "auto" The maximum number of threads to use for parallel processing. If set to "auto", the max_threads will be set to std::thread::hardware_concurrency() / 2. """ self._init_handle(_core.BatchGrammarMatcher(max_threads)) def batch_fill_next_token_bitmask( self, matchers: List["GrammarMatcher"], bitmask: ArrayLike, indices: Optional[List[int]] = None, debug_print: bool = False, ) -> None: """Fill the next token bitmask for multiple matchers. Parameters ---------- matchers : List[GrammarMatcher] The list of matchers to fill the bitmask for. bitmask : ArrayLike Must be a 2-dimensional int32 tensor with shape (bitmask_batch_size, bitmask_size). Bitmask_batch_size could be larger than the actual batch size to allow padding. Bitmask_size equals to ceil(vocab_size/32), and could be computed through xgrammar.allocate_token_bitmask. indices : Optional[List[int]], default: None A list of indices to specify which rows in the bitmask to fill. If None, fill the bitmask [0:len(matchers))]. debug_print : bool, default: False Whether to print information about generated bitmask. Helpful for debugging. Raises ------ RuntimeError If the bitmask is invalid (not on CPU, not int32, shape mismatch). """ matcher_handles = [matcher._handle for matcher in matchers] self._handle.batch_fill_next_token_bitmask(matcher_handles, bitmask, indices, debug_print) @staticmethod def batch_accept_token( matchers: List["GrammarMatcher"], tokens: List[int], debug_print: bool = False ) -> List[bool]: """Accept a batch of tokens for multiple matchers. Parameters ---------- matchers : List[GrammarMatcher] The list of matchers to accept tokens for. tokens : List[int] The list of tokens to accept. debug_print : bool, default: False Whether to print information about generated bitmask. Helpful for debugging. Returns ------- accepted : List[bool] A list of booleans indicating whether each token was accepted by its corresponding matcher. Raises ------ RuntimeError If the sizes of matchers and tokens do not match. """ matcher_handles = [matcher._handle for matcher in matchers] result = _core.BatchGrammarMatcher.batch_accept_token(matcher_handles, tokens, debug_print) return [bool(result[i]) for i in range(len(result))] @staticmethod def batch_accept_string( matchers: List["GrammarMatcher"], strings: List[Union[str, bytes]], debug_print: bool = False, ) -> List[bool]: """Accept a batch of strings for multiple matchers. Parameters ---------- matchers : List[GrammarMatcher] The list of matchers to accept tokens for. strings : List[Union[str, bytes]] The list of strings to accept. debug_print : bool, default: False Whether to print information about generated bitmask. Helpful for debugging. Returns ------- accepted : List[bool] A list of booleans indicating whether each string was accepted by its corresponding matcher. Raises ------ RuntimeError If the sizes of matchers and strings do not match. """ matcher_handles = [matcher._handle for matcher in matchers] result = _core.BatchGrammarMatcher.batch_accept_string( matcher_handles, strings, debug_print ) return [bool(result[i]) for i in range(len(result))] @staticmethod def batch_rollback( matchers: List["GrammarMatcher"], num_tokens: Union[List[int], int] = 1 ) -> None: """Rollback a batch of matchers by the given number of tokens. Parameters ---------- matchers : List[GrammarMatcher] The list of matchers to rollback. num_tokens : List[int] | int, default: 1 The number of tokens to rollback for each matcher. If an integer is provided, it will be used for all matchers; if a list is provided, it must have the same length as matchers. Raises ------ RuntimeError If the sizes of matchers and num_tokens do not match. """ if isinstance(num_tokens, int): num_tokens = [num_tokens] * len(matchers) matcher_handles = [matcher._handle for matcher in matchers] _core.BatchGrammarMatcher.batch_rollback(matcher_handles, num_tokens) xgrammar-0.2.3/python/xgrammar/openai_tool_call_schema.py000066400000000000000000000214471521764210300236770ustar00rootroot00000000000000"""OpenAI Chat Completions API tool call schema definitions. Pydantic models aligned with the official openai-python SDK (generated from OpenAPI spec by Stainless). This module models the Chat Completions tool-call shape. The Responses API uses different flat tool and tool_choice shapes; see the class docstrings below. """ # Adapted from openai-python, licensed under Apache License 2.0. # Original project: https://github.com/openai/openai-python # Modified for XGrammar. from typing import Any, Dict, List, Literal, Optional, Union from pydantic import BaseModel # ============================================================ # tools: list[FunctionToolParam] # ============================================================ class FunctionDefinition(BaseModel): """A JSON-Schema-based function definition. Corresponds to ``openai.types.shared_params.FunctionDefinition``. In Chat Completions this object is nested under ``tools[].function``. In the Responses API, the function tool fields are flat on the tool object itself and correspond to ``openai.types.responses.FunctionToolParam``. """ name: str """The name of the function to be called. Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length of 64. """ description: Optional[str] = None """A description of what the function does, used by the model to choose when and how to call the function. """ parameters: Optional[Dict[str, Any]] = None """The parameters the functions accepts, described as a JSON Schema object. If omitted or set to ``None``, the generated function arguments will be unconstrained. """ strict: Optional[bool] = None """Whether to enable strict schema adherence when generating the function call. If set to true, the model will follow the exact schema defined in the ``parameters`` field. """ class FunctionToolParam(BaseModel): """A function tool that can be used to generate a response. Corresponds to ``openai.types.chat.ChatCompletionFunctionToolParam``. Chat Completions shape:: {"type": "function", "function": {"name": "...", "parameters": {...}}} Responses API shape is flat and corresponds to ``openai.types.responses.FunctionToolParam``:: {"type": "function", "name": "...", "parameters": {...}} """ type: Literal["function"] = "function" """The type of the tool. Currently, only ``function`` is supported.""" function: FunctionDefinition """The function definition.""" class BuiltinToolParam(BaseModel): """A builtin tool whose output should be constrained. This mirrors hosted/server tool declarations used by APIs such as OpenAI Responses or Anthropic Messages. ``type`` is the provider-facing builtin tool type. ``name`` is the tool name that appears in the model output; when omitted, callers should use ``type`` as the output name. ``parameters`` is the JSON schema required by XGrammar and serving engines for constrained decoding. """ type: str """The provider-facing builtin tool type.""" name: Optional[str] = None """The output tool name for model. XGrammar-specific field. Used to constrain the tool name. Use this when the model emits a tool name that differs from the provider ``type``. For example, an OpenAI-style ``web_search_preview`` builtin may be emitted as ``browser.search`` by a Harmony-style model. Defaults to ``type`` when omitted. """ parameters: Optional[Dict[str, Any]] = None """Argument schema for the builtin tool. XGrammar-specific field. Hosted/server tool APIs often do not require users to provide this schema, but XGrammar and serving engines need it to constrain the arguments emitted by the model. """ ToolParam = Union[FunctionToolParam, BuiltinToolParam] """A function or builtin tool accepted by builtin structural tag APIs.""" # ============================================================ # tool_choice: ToolChoiceOptionParam # ============================================================ class NamedToolChoiceFunction(BaseModel): """The nested function reference used by Chat Completions named tool choice. Corresponds to ``openai.types.chat.chat_completion_named_tool_choice_param.Function``. Responses API named function choice is flat and corresponds to ``openai.types.responses.ToolChoiceFunctionParam``; it uses ``{"type": "function", "name": "..."}`` without this nested object. """ name: str """The name of the function to call.""" class NamedToolChoiceParam(BaseModel): """Specifies a tool the model should use. Use to force the model to call a specific function. Corresponds to ``openai.types.chat.ChatCompletionNamedToolChoiceParam``. Chat Completions shape:: {"type": "function", "function": {"name": "..."}} Responses API shape is flat and corresponds to ``openai.types.responses.ToolChoiceFunctionParam``:: {"type": "function", "name": "..."} """ type: Literal["function"] = "function" """For function calling, the type is always ``function``.""" function: NamedToolChoiceFunction """The function to call.""" class BuiltinToolChoiceParam(BaseModel): """Specifies a builtin tool the model should use. ``type`` matches the builtin tool type in ``tools``. Matching is based on ``type``; ``name`` is accepted for API compatibility but is not used for matching. """ type: str """The builtin tool type.""" name: Optional[str] = None """Optional model-output tool name. Builtin choices are matched by ``type``.""" class AllowedToolRef(BaseModel): """A reference to a function or builtin tool allowed in this turn. Corresponds to one item in ``openai.types.chat.ChatCompletionAllowedToolsParam.tools``. Chat Completions tool refs are nested, for example ``{"type": "function", "function": {"name": "get_weather"}}``. Responses API refs are flat dictionaries in ``openai.types.responses.ToolChoiceAllowedParam.tools``, for example ``{"type": "function", "name": "get_weather"}``. """ type: str """The allowed tool type.""" function: Optional[NamedToolChoiceFunction] = None """The function reference when ``type`` is ``"function"``.""" name: Optional[str] = None """Optional model-output builtin tool name. Builtin refs are matched by ``type``.""" class AllowedToolsParam(BaseModel): """Constrains the tools available to the model to a pre-defined set. Corresponds to ``openai.types.chat.ChatCompletionAllowedToolsParam``. In Chat Completions this object is nested under ``ChatCompletionAllowedToolChoiceParam.allowed_tools``. In the Responses API, the equivalent fields are directly on ``openai.types.responses.ToolChoiceAllowedParam``. """ mode: Literal["auto", "required"] """Constrains the tools available to the model to a pre-defined set. ``auto`` allows the model to pick from among the allowed tools and generate a message. ``required`` requires the model to call one or more of the allowed tools. """ tools: List[AllowedToolRef] """A list of tool definitions that the model should be allowed to call. For the Chat Completions API, the list of tool definitions might look like: .. code-block:: json [ { "type": "function", "function": { "name": "get_weather" } }, { "type": "function", "function": { "name": "get_time" } } ] """ class AllowedToolChoiceParam(BaseModel): """Constrains the tools available to the model to a pre-defined set. Corresponds to ``openai.types.chat.ChatCompletionAllowedToolChoiceParam``. Chat Completions shape:: {"type": "allowed_tools", "allowed_tools": {"mode": "...", "tools": [...]}} Responses API shape is flat and corresponds to ``openai.types.responses.ToolChoiceAllowedParam``:: {"type": "allowed_tools", "mode": "...", "tools": [...]} """ type: Literal["allowed_tools"] = "allowed_tools" """Allowed tool configuration type. Always ``allowed_tools``.""" allowed_tools: AllowedToolsParam """Constrains the tools available to the model to a pre-defined set.""" ToolChoiceOptionParam = Union[ Literal["none", "auto", "required"], NamedToolChoiceParam, AllowedToolChoiceParam, BuiltinToolChoiceParam, ] """Controls which (if any) tool is called by the model. ``none`` means the model will not call any tool and instead generates a message. ``auto`` means the model can pick between generating a message or calling one or more tools. ``required`` means the model must call one or more tools. Specifying a particular tool via ``{"type": "function", "function": {"name": "my_function"}}`` forces the model to call that tool. Corresponds to openai.types.chat.ChatCompletionToolChoiceOptionParam. """ xgrammar-0.2.3/python/xgrammar/py.typed000066400000000000000000000000001521764210300201570ustar00rootroot00000000000000xgrammar-0.2.3/python/xgrammar/structural_tag.py000066400000000000000000000471311521764210300221150ustar00rootroot00000000000000"""Defines all structural tag formats.""" import json from typing import Any, Dict, List, Literal, Tuple, Type, Union try: # Python 3.9+ from typing import Annotated except ImportError: # Python 3.8 from typing_extensions import Annotated from pydantic import BaseModel, Field # ---------- Basic Formats ---------- class ConstStringFormat(BaseModel): """A format that matches a constant string.""" type: Literal["const_string"] = "const_string" """The type of the format.""" value: str """The constant string.""" class JSONSchemaFormat(BaseModel): """A format that matches a JSON schema.""" type: Literal["json_schema"] = "json_schema" """The type of the format.""" json_schema: Union[bool, Dict[str, Any]] """The JSON schema.""" style: Literal["json", "qwen_xml", "minimax_xml", "deepseek_xml", "glm_xml"] = "json" """How to parse the content. Valid values: \"json\" (standard JSON), \"qwen_xml\" (Qwen XML: value), \"minimax_xml\" (MiniMax XML: value), \"deepseek_xml\" (DeepSeek XML(DeepSeek-v3.2): <{dsml_token}parameter name=\"key\" string=\"true|false\">value), \"glm_xml\" (GLM XML: keyvalue).""" any_order: bool = False """Whether object properties may appear in any order. - False (default): properties follow the schema's declared order, fully validated (required keys present, no duplicates). - True: properties may appear in any order; only key validity and each key's value schema are enforced. Key presence and uniqueness are not checked, so required keys may be missing and keys may repeat. The entry count is bounded to ``[max(minProperties, n_required), maxProperties]`` (unbounded when maxProperties is unset). Applies to every object, nested included.""" class AnyTextFormat(BaseModel): """A format that matches any text.""" type: Literal["any_text"] = "any_text" """The type of the format.""" excludes: List[str] = [] """List of strings that should not appear in the matched text.""" class TokenFormat(BaseModel): """A format that matches a single token by ID or string representation.""" type: Literal["token"] = "token" """The type of the format.""" token: Union[int, str] """The token ID (int) or token string (str).""" class ExcludeTokenFormat(BaseModel): """A format that matches a single token, excluding those in the given set.""" type: Literal["exclude_token"] = "exclude_token" """The type of the format.""" exclude_tokens: List[Union[int, str]] = [] """List of token IDs or strings to exclude.""" class AnyTokensFormat(BaseModel): """A format that matches zero or more tokens, excluding those in the given set.""" type: Literal["any_tokens"] = "any_tokens" """The type of the format.""" exclude_tokens: List[Union[int, str]] = [] """List of token IDs or strings to exclude.""" class GrammarFormat(BaseModel): """A format that matches an ebnf grammar.""" type: Literal["grammar"] = "grammar" """The type of the format.""" grammar: str """The ebnf grammar.""" class RegexFormat(BaseModel): """A format that matches a regex pattern.""" type: Literal["regex"] = "regex" """The type of the format.""" pattern: str """The regex pattern.""" # ---------- Combinatorial Formats ---------- class SequenceFormat(BaseModel): """A format that matches a sequence of formats.""" type: Literal["sequence"] = "sequence" """The type of the format.""" elements: List["Format"] """The elements of the sequence.""" class OrFormat(BaseModel): """A format that matches one of the formats.""" type: Literal["or"] = "or" """The type of the format.""" elements: List["Format"] """The elements of the or.""" class TagFormat(BaseModel): """A format that matches a tag: ``begin content end``. The ``end`` field can be a single string or a list of possible end strings. When multiple end strings are provided, any of them will be accepted as a valid ending for the tag. Examples -------- Single end string: .. code-block:: python TagFormat(begin="", content=..., end="") Multiple end strings: .. code-block:: python TagFormat(begin="", content=..., end=["", ""]) """ type: Literal["tag"] = "tag" """The type of the format.""" begin: Union[str, TokenFormat] """The begin tag. Can be a string or a TokenFormat.""" content: "Format" """The content of the tag. It can be any of the formats.""" end: Union[str, List[str], TokenFormat] """The end tag(s). Can be a string, list of strings, or a TokenFormat.""" class TriggeredTagsFormat(BaseModel): """A format that matches triggered tags. It can allow any output until a trigger is encountered, then dispatch to the corresponding tag; when the end tag is encountered, the grammar will allow any following output, until the next trigger is encountered. Each tag should be matched by exactly one trigger. "matching" means the trigger should be a prefix of the begin tag. Tags must use **string** ``begin`` fields, not ``TokenFormat``. For token-level dispatch, use ``TokenTriggeredTagsFormat`` instead. Examples -------- .. code-block:: python structural_tag = TriggeredTagsFormat( triggers=["{"name": "John", "age": 30} {"name": "Jane", "age": 25} any_text{"name": "John", "age": 30}any_text1{"name": "Jane", "age": 25}any_text2 """ type: Literal["triggered_tags"] = "triggered_tags" """The type of the format.""" triggers: List[str] """The triggers of the triggered tags.""" tags: List[TagFormat] """The tags of the triggered tags.""" at_least_one: bool = False """Whether at least one of the tags must be generated.""" stop_after_first: bool = False """Whether to stop after the first tag is generated.""" excludes: List[str] = [] """List of strings that should not appear in the matched text.""" class TokenTriggeredTagsFormat(BaseModel): """A format that dispatches to tags based on token-level triggers. Similar to TriggeredTagsFormat but uses token IDs instead of string triggers. Tags must use ``TokenFormat`` ``begin`` fields, not strings. For string-level dispatch, use ``TriggeredTagsFormat`` instead. """ type: Literal["token_triggered_tags"] = "token_triggered_tags" """The type of the format.""" trigger_tokens: List[Union[int, str]] """The trigger token IDs or strings.""" tags: List[TagFormat] """The tags to dispatch to.""" exclude_tokens: List[Union[int, str]] = [] """List of token IDs or strings to exclude.""" at_least_one: bool = False """Whether at least one tag must be generated.""" stop_after_first: bool = False """Whether to stop after the first tag is generated.""" class TagsWithSeparatorFormat(BaseModel): """A format that matches a tags with separator. It can match zero, one, or more tags, separated by the separator, with no other text allowed. Examples -------- .. code-block:: python structural_tag = TagsWithSeparatorFormat( tags=[ TagFormat(begin="", content=JSONSchemaFormat(json_schema=...), end=""), TagFormat(begin="", content=JSONSchemaFormat(json_schema=...), end=""), ], separator=",", at_least_one=False, stop_after_first=False, ) The above structural tag can accept an empty string, or the following outputs:: {"name": "John", "age": 30} {"name": "John", "age": 30},{"name": "Jane", "age": 25} {"name": "John", "age": 30},{"name": "Jane", "age": 25},{"name": "John", "age": 30} """ type: Literal["tags_with_separator"] = "tags_with_separator" """The type of the format.""" tags: List[TagFormat] """The tags of the tags with separator.""" separator: str """The separator of the tags with separator.""" at_least_one: bool = False """Whether at least one of the tags must be matched.""" stop_after_first: bool = False """Whether to stop after the first tag is matched.""" class OptionalFormat(BaseModel): """A format that matches the content 0 or 1 time (EBNF optional). Semantics: the inner format may appear once or not at all. """ type: Literal["optional"] = "optional" """The type of the format.""" content: "Format" """The format that may appear 0 or 1 time.""" class PlusFormat(BaseModel): """A format that matches the content 1 or more times (EBNF plus). Semantics: the inner format must appear at least once. """ type: Literal["plus"] = "plus" """The type of the format.""" content: "Format" """The format that must appear at least once.""" class StarFormat(BaseModel): """A format that matches the content 0 or more times (EBNF star). Semantics: the inner format may appear any number of times. """ type: Literal["star"] = "star" """The type of the format.""" content: "Format" """The format that may appear 0 or more times.""" class RepeatFormat(BaseModel): """A format that matches the content between min and max times (inclusive). Use max=-1 for unbounded upper limit (e.g. "at least min times"). """ type: Literal["repeat"] = "repeat" """The type of the format.""" min: int """Minimum number of occurrences (inclusive).""" max: int """Maximum number of occurrences (inclusive). Use -1 for unbounded.""" content: "Format" """The format that is repeated.""" class DispatchFormat(BaseModel): """Matches certain patterns in free-form text. When certain strings are generated, the following content must follow the corresponding format. Certain strings can be excluded from being generated in the free-form part. The user specifies a list of (pattern, formats). The LLM can generate any free-form text, but when a pattern is matched in the text, the following output must follow the corresponding format. The ``loop`` field controls whether the matching of this structure ends after accepting the first pattern and format. If False, it ends. If true, the matching continues, and this format will continue to allow free-form text and detect the next pattern to be generated. The ``excludes`` field controls the strings that cannot be generated in the free-form text. It can also control the end of the format: ``SequenceFormat(DispatchFormat(..., excludes=""), ConstStringFormat(""))`` or ``TagFormat(begin=..., content=DispatchFormat(..., excludes=""), end="")`` ends the matching of the format when LLM generates . """ type: Literal["dispatch"] = "dispatch" """The type of the format.""" rules: List[Tuple[str, "Format"]] """List of ``(pattern, content format)`` pairs.""" loop: bool = True """If true, after handling one dispatched format, it will continue to allow free-form text and match the next pattern. Otherwise, the matching of this format ends after handling the first dispatched format.""" excludes: List[str] = [] """List of strings that must not appear in the free-form text.""" class TokenDispatchFormat(BaseModel): """Matches certain patterns in free-form text. When certain tokens are generated, the following content must follow the corresponding format. Certain tokens can be excluded from being generated in the free-form part. The user specifies a list of (pattern token, formats). The LLM can generate any free-form text, but when a pattern is matched in the text, the following output must follow the corresponding format. The ``loop`` field controls whether the matching of this structure ends after accepting the first pattern and format. If False, it ends. If true, the matching continues, and this format will continue to allow free-form text and detect the next pattern to be generated. The ``excludes`` field controls the strings that cannot be generated in the free-form text. It can also control the end of the format: ``SequenceFormat(DispatchFormat(..., excludes=""), ConstStringFormat(""))`` or ``TagFormat(begin=..., content=DispatchFormat(..., excludes=""), end="")`` ends the matching of the format when LLM generates . """ type: Literal["token_dispatch"] = "token_dispatch" """The type of the format.""" rules: List[Tuple[Union[int, str], "Format"]] """List of ``(pattern token, content format)`` pairs. Pattern is token ID or token string.""" loop: bool = True """If true, after one dispatched format, it will continue to allow free-form text and match the next pattern. Otherwise, the matching of this format ends after handling the first dispatched format.""" exclude_tokens: List[Union[int, str]] = [] """List of tokens that must not appear in the free-form text.""" # ---------- Deprecated Formats ---------- class QwenXMLParameterFormat(BaseModel): """Deprecated. Use :class:`JSONSchemaFormat` with ``style="qwen_xml"`` instead. This format remains available so existing serialized structural tags with ``{"type": "qwen_xml_parameter"}`` can still be loaded. Examples -------- Use the replacement format for new structural tags: .. code-block:: python structural_tag = JSONSchemaFormat( json_schema={ "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], }, style="qwen_xml", ) The above structural tag can accept the following outputs:: Bob100 "Bob<"100 """ type: Literal["qwen_xml_parameter"] = "qwen_xml_parameter" """The type of the deprecated format.""" json_schema: Union[bool, Dict[str, Any]] """The JSON schema for the parameters of the function calling.""" # ---------- Discriminated Union ---------- Format = Annotated[ Union[ AnyTextFormat, ConstStringFormat, JSONSchemaFormat, GrammarFormat, RegexFormat, QwenXMLParameterFormat, OrFormat, SequenceFormat, TagFormat, TriggeredTagsFormat, TokenTriggeredTagsFormat, TagsWithSeparatorFormat, OptionalFormat, PlusFormat, StarFormat, TokenFormat, ExcludeTokenFormat, AnyTokensFormat, RepeatFormat, DispatchFormat, TokenDispatchFormat, ], Field(discriminator="type"), ] """Union of all structural tag formats.""" # Solve forward references if hasattr(BaseModel, "model_rebuild"): SequenceFormat.model_rebuild() TagFormat.model_rebuild() TriggeredTagsFormat.model_rebuild() TokenTriggeredTagsFormat.model_rebuild() TagsWithSeparatorFormat.model_rebuild() OptionalFormat.model_rebuild() PlusFormat.model_rebuild() StarFormat.model_rebuild() RepeatFormat.model_rebuild() DispatchFormat.model_rebuild() TokenDispatchFormat.model_rebuild() elif hasattr(BaseModel, "update_forward_refs"): SequenceFormat.update_forward_refs() TagFormat.update_forward_refs() TriggeredTagsFormat.update_forward_refs() TokenTriggeredTagsFormat.update_forward_refs() TagsWithSeparatorFormat.update_forward_refs() OptionalFormat.update_forward_refs() PlusFormat.update_forward_refs() StarFormat.update_forward_refs() RepeatFormat.update_forward_refs() DispatchFormat.update_forward_refs() TokenDispatchFormat.update_forward_refs() else: raise RuntimeError("Unsupported pydantic version") # ---------- Top Level ---------- class StructuralTagItem(BaseModel): """Deprecated. Definition of a structural tag item. See :meth:`xgrammar.Grammar.from_structural_tag` for more details. """ begin: str """The begin tag.""" schema_: Union[str, Type[BaseModel], Dict[str, Any]] = Field(alias="schema") """The schema.""" end: str """The end tag.""" class StructuralTag(BaseModel): """ Describes a complete structural tag structure. It corresponds to ``"response_format": {"type": "structural_tag", "format": {...}}`` in API. """ type: Literal["structural_tag"] = "structural_tag" """The type must be "structural_tag".""" format: Format """The format of the structural tag. Could be any of the structural tag formats.""" @staticmethod def from_legacy_structural_tag( tags: List[StructuralTagItem], triggers: List[str] ) -> "StructuralTag": """Convert a legacy structural tag item to a structural tag.""" return StructuralTag( type="structural_tag", format=TriggeredTagsFormat( type="triggered_tags", triggers=triggers, tags=[ TagFormat( begin=tag.begin, content=JSONSchemaFormat( json_schema=( json.loads(tag.schema_) if isinstance(tag.schema_, str) else ( tag.schema_.model_json_schema() if isinstance(tag.schema_, type) and issubclass(tag.schema_, BaseModel) else tag.schema_ ) ) ), end=tag.end, ) for tag in tags ], ), ) @staticmethod def from_json(json_str: Union[str, Dict[str, Any]]) -> "StructuralTag": """Convert a JSON string to a structural tag.""" if isinstance(json_str, str): return StructuralTag.model_validate_json(json_str) elif isinstance(json_str, dict): return StructuralTag.model_validate(json_str) else: raise ValueError("Invalid JSON string or dictionary") __all__ = [ "ConstStringFormat", "JSONSchemaFormat", "QwenXMLParameterFormat", "AnyTextFormat", "GrammarFormat", "RegexFormat", "TokenFormat", "ExcludeTokenFormat", "AnyTokensFormat", "SequenceFormat", "OrFormat", "TagFormat", "TriggeredTagsFormat", "TokenTriggeredTagsFormat", "TagsWithSeparatorFormat", "OptionalFormat", "PlusFormat", "StarFormat", "RepeatFormat", "Format", "StructuralTagItem", "StructuralTag", ] xgrammar-0.2.3/python/xgrammar/testing.py000066400000000000000000000374531521764210300205350ustar00rootroot00000000000000"""Testing utilities. The APIs in this module are used for testing and debugging and are prone to change. Don't use them in production.""" import time from typing import Any, Dict, List, Optional, Tuple, Type, Union import torch from pydantic import BaseModel from .base import _core from .compiler import CompiledGrammar, GrammarCompiler from .grammar import Grammar, _convert_schema_to_str from .matcher import GrammarMatcher, bitmask_dtype from .tokenizer_info import TokenizerInfo def _json_schema_to_ebnf( schema: Union[str, Type[BaseModel], Dict[str, Any]], *, any_whitespace: bool = True, indent: Optional[int] = None, separators: Optional[Tuple[str, str]] = None, max_whitespace_cnt: Optional[int] = None, strict_mode: bool = True, any_order: bool = False, ) -> str: """Convert JSON schema string to BNF grammar string. For test purposes. Parameters ---------- schema : Union[str, Type[BaseModel], Dict[str, Any]] The schema string or Pydantic model or JSON schema dict. indent : Optional[int], default: None The number of spaces for indentation. If None, the output will be in one line. separators : Optional[Tuple[str, str]], default: None Two separators used in the schema: comma and colon. Examples: (",", ":"), (", ", ": "). If None, the default separators will be used: (",", ": ") when the indent is not None, and (", ", ": ") otherwise. strict_mode : bool, default: True Whether to use strict mode. In strict mode, the generated grammar will not allow properties and items that is not specified in the schema. This is equivalent to setting unevaluatedProperties and unevaluatedItems to false. This helps LLM to generate accurate output in the grammar-guided generation with JSON schema. max_whitespace_cnt : Optional[int], default: None The maximum number of whitespace characters allowed between elements, such like keys, values, separators and so on. If None, there is no limit on the number of whitespace characters. If specified, it will limit the number of whitespace characters to at most max_whitespace_cnt. It should be a positive integer. Returns ------- bnf_string : str The BNF grammar string. """ schema_str = _convert_schema_to_str(schema) return _core.testing._json_schema_to_ebnf( schema_str, any_whitespace, indent, separators, strict_mode, max_whitespace_cnt, any_order ) def _regex_to_ebnf(regex: str, with_rule_name: bool = True) -> str: r"""Convert a regex string to BNF grammar string. For test purposes. The regex grammar follows the syntax in JavaScript (ECMA 262). Check https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Regular_expressions for a tutorial. Currently the following features are not supported: 1. Backreference (\1) 2. non-capturing group, naming capture groups and assertions ((?...)) 3. Unicode character class escape (\p{...}) 4. Word boundary (\b) 5. Unicode property escapes (\p{...}) 6. Quantifier with range {x,y}. Now user can just repeat the element as a workaround. This method is primarily intended for testing and debugging purposes. Parameters ---------- regex : str The regex string to be converted. Returns ------- bnf_string : str The BNF grammar string converted from the input regex. """ return _core.testing._regex_to_ebnf(regex, with_rule_name) def _ebnf_to_grammar_no_normalization(ebnf_string: str, root_rule_name: str = "root") -> Grammar: """Convert a BNF grammar string to a Grammar object without normalization. For test purposes. The result grammar cannot be compiled / used in GrammarMatcher. Parameters ---------- ebnf_string : str The BNF grammar string to be converted. Returns ------- grammar : Grammar The unnormalized Grammar object converted from the input BNF grammar string. """ return Grammar._create_from_handle( _core.testing._ebnf_to_grammar_no_normalization(ebnf_string, root_rule_name) ) def _get_matcher_from_grammar(grammar: Union[Grammar, str], **kwargs) -> GrammarMatcher: """Create a GrammarMatcher from a grammar. The tokenizer info will be set to an empty TokenizerInfo. The result matcher can only accept strings, and cannot accept tokens. Parameters ---------- grammar : Union[Grammar, str] The grammar to create the matcher from. Can be either a Grammar object or a string containing EBNF grammar. Returns ------- matcher : GrammarMatcher The created grammar matcher. """ tokenizer_info = TokenizerInfo([]) grammar_compiler = GrammarCompiler(tokenizer_info, cache_enabled=False) compiled_grammar = grammar_compiler.compile_grammar(grammar) return GrammarMatcher(compiled_grammar, terminate_without_stop_token=True, **kwargs) def _is_grammar_accept_string( grammar: Union[Grammar, str], input_str: str, *, debug_print: bool = False, print_time: bool = False, require_termination: bool = True, ) -> bool: """Check if a grammar accepts a string. For test purposes. Parameters ---------- grammar : Union[Grammar, str] The grammar to check. Can be either a Grammar object or a BNF grammar string. input_str : str The input string to check. debug_print : bool, default: False Whether to print debug information during matching. print_time : bool, default: False Whether to print timing information. Returns ------- bool True if the grammar accepts the string, False otherwise. """ grammar_matcher = _get_matcher_from_grammar(grammar) if print_time: start = time.monotonic_ns() accepted = grammar_matcher.accept_string(input_str, debug_print=debug_print) if print_time: end = time.monotonic_ns() print(f"Accepting {input_str}, result: {accepted}, time: {(end - start) / 1e3} us") if not accepted: return False if not require_termination: return True return grammar_matcher.is_terminated() def _get_masked_tokens_from_bitmask( bitmask: torch.Tensor, vocab_size: int, index: int = 0 ) -> List[int]: """Get the ids of the rejected tokens from the bitmask. Mainly for debug purposes. Parameters ---------- bitmask : torch.Tensor The rejected token bitmask. Should be generated by allocate_token_bitmask and filled by fill_next_token_bitmask. Should be on CPU. index : int, default: 0 The batch index of the bitmask. For batch inference, bitmask[index] will be used. Otherwise is ignored. Returns ------- rejected_token_ids : List[int] A list of rejected token ids. """ if bitmask.device.type != "cpu": raise ValueError("bitmask should be on CPU.") if bitmask.dtype != bitmask_dtype: raise ValueError(f"bitmask should be of type {bitmask_dtype}.") return list( _core.testing._get_masked_tokens_from_bitmask( bitmask.data_ptr(), list(bitmask.shape), vocab_size, index ) ) def _is_single_token_bitmask( bitmask: torch.Tensor, vocab_size: int, index: int = 0 ) -> Tuple[bool, int]: """Check if the bitmask is a single token bitmask. Parameters ---------- bitmask : torch.Tensor The bitmask to check. Should be on CPU. vocab_size : int The size of the vocabulary. index : int, default: 0 The index of the bitmask. Returns ------- is_single_token : bool True if the bitmask is a single token bitmask, False otherwise. token_id : int The id of the token if the bitmask is a single token bitmask, -1 otherwise. """ result = _core.testing._is_single_token_bitmask( bitmask.data_ptr(), list(bitmask.shape), vocab_size, index ) return bool(result[0]), result[1] def bool_mask_to_bitmask(bool_mask: torch.Tensor) -> torch.Tensor: """Get the bitmask from bool mask. If the bool mask does not align with the 32-bit block size, it will add extra 1 paddings. Parameters ---------- bool_mask : torch.Tensor The rejected token bool mask. For each element value, True means the token is allowed, while False means the token is rejected. Returns ------- bitmask : torch.Tensor The rejected token bitmask. """ bool_mask_int32 = bool_mask.to(torch.int32) # Pad to multiple of 32 pad_size = (32 - bool_mask.shape[1] % 32) % 32 if pad_size > 0: bool_mask_int32 = torch.nn.functional.pad(bool_mask_int32, (0, pad_size), value=1) bool_mask_view = bool_mask_int32.view(bool_mask.shape[0], -1, 32) # To avoid error for overflow, we construct int64 weights and convert to int32 weights = torch.tensor( [1 << i for i in range(32)], device=bool_mask.device, dtype=torch.int64 ).to(torch.int32) bitmask = (bool_mask_view * weights).sum(dim=2) return bitmask.to(torch.int32) def bitmask_to_bool_mask(bit_mask: torch.Tensor, vocab_size: Optional[int] = None) -> torch.Tensor: """ Convert a bitmask tensor to a boolean mask tensor. Parameters ---------- bit_mask : torch.Tensor The bitmask tensor to convert. Should be on CPU and of type int32. vocab_size : Optional[int], default: None The size of the vocabulary. If provided, the output mask will be cut to this size. Returns ------- bool_mask : torch.Tensor The converted boolean mask tensor. """ # Validate input. if bit_mask.device.type != "cpu": raise ValueError("bit_mask should be on CPU.") if bit_mask.dtype != bitmask_dtype: raise ValueError("bit_mask should be of type torch.int32.") if vocab_size is None: vocab_size = bit_mask.shape[1] * 32 if vocab_size > bit_mask.shape[1] * 32: raise ValueError( "vocab_size should be less than or equal to the size represented by bit_mask." ) bool_mask = torch.zeros((bit_mask.shape[0], vocab_size), dtype=torch.bool) for i in range(vocab_size): bool_mask[:, i] = (bit_mask[:, i // 32] & (1 << (i % 32))) != 0 return bool_mask def _get_matcher_from_grammar_and_tokenizer_info( grammar: Union[Grammar, str], tokenizer_info: Optional[TokenizerInfo] = None, **kwargs ) -> GrammarMatcher: """Create a GrammarMatcher from a grammar and tokenizer info. Parameters ---------- grammar : Union[Grammar, str] The grammar to create the matcher from. Can be either a Grammar object or a string containing EBNF grammar. tokenizer_info : Optional[TokenizerInfo], default: None Information about the tokenizer to use with this grammar. If None, an empty TokenizerInfo will be created. **kwargs Additional keyword arguments to pass to the GrammarMatcher constructor. Returns ------- matcher : GrammarMatcher The created grammar matcher. """ if tokenizer_info is None: tokenizer_info = TokenizerInfo([]) grammar_compiler = GrammarCompiler(tokenizer_info, cache_enabled=False) compiled_grammar = grammar_compiler.compile_grammar(grammar) return GrammarMatcher(compiled_grammar, **kwargs) def _get_allow_empty_rule_ids(compiled_grammar: CompiledGrammar) -> List[int]: return list(_core.testing._get_allow_empty_rule_ids(compiled_grammar._handle)) def _generate_range_regex(start: Optional[int] = None, end: Optional[int] = None) -> str: return _core.testing._generate_range_regex(start, end) def _generate_float_regex( start: Optional[float] = None, end: Optional[float] = None, exclusive_start: bool = False, exclusive_end: bool = False, ) -> str: return _core.testing._generate_float_regex(start, end, exclusive_start, exclusive_end) def _print_grammar_fsms(grammar: Grammar) -> str: """Print the FSMs of the grammar. Now the fsms are initialized in the grammar compilation process.""" return _core.testing._print_grammar_fsms(grammar._handle) def _qwen_xml_tool_calling_to_ebnf( schema: Union[str, Type[BaseModel], Dict[str, Any]], any_order: bool = False ) -> str: """Convert Qwen XML tool calling schema to EBNF.""" schema_str = _convert_schema_to_str(schema) return _core.testing._qwen_xml_tool_calling_to_ebnf(schema_str, any_order) def _minimax_xml_tool_calling_to_ebnf( schema: Union[str, Type[BaseModel], Dict[str, Any]], any_order: bool = False ) -> str: """Convert MiniMax XML tool calling schema to EBNF.""" schema_str = _convert_schema_to_str(schema) return _core.testing._minimax_xml_tool_calling_to_ebnf(schema_str, any_order) def _deepseek_xml_tool_calling_to_ebnf( schema: Union[str, Type[BaseModel], Dict[str, Any]], any_order: bool = False ) -> str: """Convert DeepSeek XML tool calling schema to EBNF.""" schema_str = _convert_schema_to_str(schema) return _core.testing._deepseek_xml_tool_calling_to_ebnf(schema_str, any_order) def _glm_xml_tool_calling_to_ebnf( schema: Union[str, Type[BaseModel], Dict[str, Any]], any_order: bool = False ) -> str: """Convert GLM XML tool calling schema to EBNF.""" schema_str = _convert_schema_to_str(schema) return _core.testing._glm_xml_tool_calling_to_ebnf(schema_str, any_order) def _traverse_draft_tree( retrieve_next_token: torch.Tensor, retrieve_next_sibling: torch.Tensor, draft_tokens: torch.Tensor, matcher: "GrammarMatcher", allocate_token_bitmask: torch.Tensor, time_threshold: float = -1.0, ) -> bool: """Backward-compatible wrapper for :meth:`GrammarMatcher.traverse_draft_tree`. Prefer :meth:`xgrammar.GrammarMatcher.traverse_draft_tree` in new code. """ return matcher.traverse_draft_tree( retrieve_next_token, retrieve_next_sibling, draft_tokens, allocate_token_bitmask, time_threshold, ) class GrammarFunctor: """A utility class for transforming grammars. These methods are called during grammar parsing. For test purposes.""" @staticmethod def structure_normalizer(grammar: Grammar) -> Grammar: """Normalize the structure of the grammar.""" return Grammar._create_from_handle( _core.testing.grammar_functor.structure_normalizer(grammar._handle) ) @staticmethod def rule_inliner(grammar: Grammar) -> Grammar: """Inline some rule references in the grammar.""" return Grammar._create_from_handle( _core.testing.grammar_functor.rule_inliner(grammar._handle) ) @staticmethod def byte_string_fuser(grammar: Grammar) -> Grammar: """Fuse the byte string elements in the grammar.""" return Grammar._create_from_handle( _core.testing.grammar_functor.byte_string_fuser(grammar._handle) ) @staticmethod def dead_code_eliminator(grammar: Grammar) -> Grammar: """Eliminate the not referenced rules in the grammar.""" return Grammar._create_from_handle( _core.testing.grammar_functor.dead_code_eliminator(grammar._handle) ) @staticmethod def lookahead_assertion_analyzer(grammar: Grammar) -> Grammar: """Analyze and add lookahead assertions in the grammar.""" return Grammar._create_from_handle( _core.testing.grammar_functor.lookahead_assertion_analyzer(grammar._handle) ) @staticmethod def grammar_optimizer(grammar: Grammar) -> Grammar: """Optimize the grammar.""" return Grammar._create_from_handle( _core.testing.grammar_functor.grammar_optimizer(grammar._handle) ) @staticmethod def repetition_normalizer(grammar: Grammar) -> Grammar: """Normalize the repetition expression.""" return Grammar._create_from_handle( _core.testing.grammar_functor.repetition_normalizer(grammar._handle) ) xgrammar-0.2.3/python/xgrammar/tokenizer_info.py000066400000000000000000000440561521764210300221020ustar00rootroot00000000000000"""This module provides the tokenizer info class to handle the tokenizer information.""" import json import warnings from enum import Enum from typing import Any, Dict, List, Optional, Union try: import sentencepiece except ImportError: sentencepiece = None try: import tiktoken except ImportError: tiktoken = None from transformers import PreTrainedTokenizerBase, PreTrainedTokenizerFast from .base import XGRObject, _core class VocabType(Enum): """The type of the vocabulary. Used in TokenizerInfo. XGrammar supports three types of vocabularies: RAW, BYTE_FALLBACK, BYTE_LEVEL. """ RAW = 0 """The vocabulary is in the raw format. The tokens in the vocabulary are kept in their original form without any processing. This kind of tokenizer includes the tiktoken tokenizer, e.g. microsoft/Phi-3-small-8k-instruct, Qwen/Qwen-7B-Chat, etc. """ BYTE_FALLBACK = 1 r"""The vocabulary used in the byte fallback BPE tokenizer. The tokens are encoded through the byte-fallback conversion. E.g. "\u001b" -> "<0x1B>", " apple" -> "▁apple". This kind of tokenizer includes meta-llama/Llama-2-7b-chat, microsoft/Phi-3.5-mini-instruct, etc. """ BYTE_LEVEL = 2 """The vocabulary used in the byte level BPE tokenizer. The tokens are encoded through the byte-to-unicode conversion, as in https://github.com/huggingface/transformers/blob/87be06ca77166e6a6215eee5a990ab9f07238a18/src/transformers/models/gpt2/tokenization_gpt2.py#L38-L59 This kind of tokenizer includes meta-llama/Meta-Llama-3-8B-Instruct, meta-llama/Meta-Llama-3.1-8B-Instruct, etc. """ class TokenizerInfo(XGRObject): """The tokenizer info contains the vocabulary, the type of the vocabulary, and necessary information for the grammar-guided generation. Note that although some tokenizers will encode the tokens in a special format, e.g. "<0x1B>" for "\u001b" in the ByteFallback tokenizer, and "Ġ" for " " in the Byte-Level BPE tokenizer, TokenizerInfo always decodes the vocabulary to the original format (e.g. "\u001b" and " "). Also note that some models (e.g. Phi-3 and Deepseek-V2) may pad the vocabulary to a multiple of 32. In this case, the model's vocab_size is larger than the tokenizer's vocabulary size. Please pass the model's vocab_size to the vocab_size parameter in the constructor, because this information is used to determine the size of the token mask. """ def __init__( self, encoded_vocab: Union[List[bytes], List[str]], vocab_type: VocabType = VocabType.RAW, *, vocab_size: Optional[int] = None, stop_token_ids: Optional[Union[List[int], int]] = None, add_prefix_space: bool = False, ) -> None: """Construct the tokenizer info. Parameters ---------- encoded_vocab : Union[List[bytes], List[str]] The encoded vocabulary of the tokenizer. vocab_type : VocabType, default: VocabType.RAW The type of the vocabulary. See also VocabType. vocab_size : Optional[int], default: None The size of the vocabulary. If not provided, the vocabulary size will be len(encoded_vocab). stop_token_ids : Optional[List[int]], default: None The stop token ids. If not provided, the stop token ids will be auto detected (but may not be correct). add_prefix_space : bool, default: False Whether the tokenizer will prepend a space before the text in the tokenization process. """ if isinstance(stop_token_ids, int): stop_token_ids = [stop_token_ids] self._init_handle( _core.TokenizerInfo( encoded_vocab, vocab_type.value, vocab_size, stop_token_ids, add_prefix_space ) ) @staticmethod def _is_tiktoken_tokenizer(tokenizer: PreTrainedTokenizerBase) -> bool: if tiktoken is None: return False # helper to check if tokenizer is a tiktoken tokenizer has_tiktoken_encoding = hasattr(tokenizer, "tokenizer") and isinstance( tokenizer.tokenizer, tiktoken.Encoding ) filename_pattern = ( hasattr(tokenizer, "vocab_files_names") and "vocab_file" in tokenizer.vocab_files_names and "tiktoken" in tokenizer.vocab_files_names["vocab_file"] ) return has_tiktoken_encoding or filename_pattern @staticmethod def _is_byte_level_tokenizer(tokenizer: PreTrainedTokenizerBase) -> bool: """Checking whether the tokenizer has byte-level whitespace conversion. Parameters ---------- tokenizer : PreTrainedTokenizerBase The huggingface tokenizer. Returns ------- is_byte_level : bool The tokenizer has byte-level whitespace conversion. """ if tiktoken is None: return False # check the tokenizer with r' ' encode new_ids = tokenizer.encode(r" ") if new_ids.__len__() < 1: return False new_tokens = tokenizer.convert_ids_to_tokens(new_ids) token = new_tokens[0] # the tokenizer has a BPE-like whitespace conversion return token == "Ġ" @staticmethod def _is_sentencepiece_tokenizer(tokenizer: PreTrainedTokenizerBase) -> bool: if sentencepiece is None: return False # helper to check if tokenizer is a sentence piece tokenizer has_sp_model_attr = hasattr(tokenizer, "sp_model") and isinstance( tokenizer.sp_model, sentencepiece.SentencePieceProcessor ) has_nested_sp_model_attr = ( hasattr(tokenizer, "tokenizer") and hasattr(tokenizer.tokenizer, "sp_model") and isinstance(tokenizer.tokenizer.sp_model, sentencepiece.SentencePieceProcessor) ) or ( # Support Teuken-7B-instruct-v0.6 hasattr(tokenizer, "tok") and isinstance(tokenizer.tok, sentencepiece.SentencePieceProcessor) ) return has_sp_model_attr or has_nested_sp_model_attr @staticmethod def from_huggingface( tokenizer: PreTrainedTokenizerBase, *, vocab_size: Optional[int] = None, stop_token_ids: Optional[Union[List[int], int]] = None, ) -> "TokenizerInfo": """Construct the tokenizer info from the huggingface tokenizer. This constructor supports various tokenizer backends, including the huggingface fast tokenizer and tiktoken tokenizer. Necessary information is automatically detected from the tokenizer. The vocab_size parameter is introduced to handle the misalignment between the model's vocab_size and the tokenizer's vocabulary size. User should pass the model's vocab_size (could be defined in the model config) here. See docs of vocab_size for more details. The stop token ids is by default the eos_token_id of the tokenizer. If there are other stop tokens, you can specify them manually. Parameters ---------- tokenizer : PreTrainedTokenizerBase The huggingface tokenizer. vocab_size : Optional[int], default: None The vocabulary size **defined by the model** (**not the tokenizer**). This equals to the vocab dimention of the model's lm_head. This is the size of the token mask. It can be: 1. the same as the tokenizer's vocabulary size. This is the most common case. 2. larger than the tokenizer's vocabulary size. This happens when the model has padding to lm_head, possibly due to aligning lm_head to the power of 2. E.g. Phi-3 and Deepseek-V2. 3. smaller than the tokenizer's vocabulary size. This happens when the tokenizer has some added tokens that will not supported by the model. E.g. Llama-3.2 Vision and Molmo-72B-0924 has padded `<|image|>` tokens, but they will not be considered in lm_head or generated by the model. model_vocab_size need to be provided for case 2 and 3. If not provided, it will be set to the tokenizer's vocabulary size. stop_token_ids : Optional[List[int]], default: None The stop token ids. If not provided, the eos_token_id of the tokenizer will be used. Returns ------- tokenizer_info : TokenizerInfo The tokenizer info. """ if isinstance(stop_token_ids, int): stop_token_ids = [stop_token_ids] if isinstance(stop_token_ids, list) and len(stop_token_ids) == 0: raise ValueError("stop_token_ids cannot be empty") try: vocab_dict = tokenizer.get_vocab() except AttributeError as e: msg = ( f"Cannot get the vocabulary of the tokenizer {type(tokenizer)}. The tokenizer " "should have a get_vocab method." ) raise ValueError(msg) from e # Some tokenizer don't have token id 0 or 1 or 2. So the max_id could be larger than the # number of tokens. max_id = max(vocab_dict.values()) tokenizer_vocab_size = max(len(vocab_dict), max_id + 1) vocab_size = vocab_size or tokenizer_vocab_size # maintain tokenizer's indexing encoded_vocab = [""] * vocab_size for token, idx in vocab_dict.items(): if idx < vocab_size: encoded_vocab[idx] = token if isinstance(tokenizer, PreTrainedTokenizerFast): # huggingface fast tokenizer # - the vocabulary is directly obtained from tokenizer.get_vocab() # (tokenizer.backend_tokenizer.to_str() may not contain the full vocab, special # tokens may be omitted) # - the vocab size is obtained from len(tokenizer.get_vocab()) or provided by user # - the vocab type and add_prefix_space are obtained from # tokenizer.backend_tokenizer.to_str() # - stop token id is provided by user, or auto detected. backend_str = tokenizer.backend_tokenizer.to_str() if stop_token_ids is None: if hasattr(tokenizer, "eos_token_id") and tokenizer.eos_token_id is not None: stop_token_ids = [tokenizer.eos_token_id] else: warnings.warn( "When constructing TokenizerInfo from a huggingface tokenizer, " "stop_token_ids is neither provided by user nor found from the tokenizer. " "It will be automatically detected." ) metadata = TokenizerInfo._detect_metadata_from_hf(backend_str) return TokenizerInfo( encoded_vocab, vocab_type=metadata["vocab_type"], vocab_size=vocab_size, stop_token_ids=stop_token_ids, add_prefix_space=metadata["add_prefix_space"], ) elif TokenizerInfo._is_tiktoken_tokenizer(tokenizer): # tiktoken tokenizer # e.g. Phi-3-small-8k-instruct, Qwen-7B-Chat, stablelm-2-12b-chat (previously) if stop_token_ids is None: if hasattr(tokenizer, "eos_token_id") and tokenizer.eos_token_id is not None: stop_token_ids = [tokenizer.eos_token_id] else: warnings.warn( "When constructing TokenizerInfo from a huggingface tokenizer, " "stop_token_ids is neither provided by user nor found from the tokenizer. " "It will be automatically detected." ) vocab_type = VocabType.RAW if TokenizerInfo._is_byte_level_tokenizer(tokenizer): # Some tiktoken tokenizers subclassed from PretrainedTokenizerBase # also perform byte-level conversion. # e.g. Kimi-K2-Instruct vocab_type = VocabType.BYTE_LEVEL return TokenizerInfo( encoded_vocab, vocab_type, vocab_size=vocab_size, stop_token_ids=stop_token_ids, add_prefix_space=False, ) elif TokenizerInfo._is_sentencepiece_tokenizer(tokenizer): # sentencepiece tokenizer # e.g. Chatglm3-6b if hasattr(tokenizer, "sp_model"): sp_model = tokenizer.sp_model elif hasattr(tokenizer, "tokenizer") and hasattr(tokenizer.tokenizer, "sp_model"): sp_model = tokenizer.tokenizer.sp_model elif hasattr(tokenizer, "tok"): sp_model = tokenizer.tok if stop_token_ids is None: if hasattr(tokenizer, "eos_token_id") and tokenizer.eos_token_id is not None: stop_token_ids = [tokenizer.eos_token_id] else: eos_id = sp_model.eos_id() if eos_id != -1: stop_token_ids = [eos_id] else: warnings.warn( "When constructing TokenizerInfo from a huggingface tokenizer, " "stop_token_ids is neither provided by user nor found from the tokenizer. " "It will be automatically detected." ) # detect vocab_type of tokenizer if "<0x0A>" in vocab_dict: vocab_type = VocabType.BYTE_FALLBACK else: vocab_type = VocabType.RAW return TokenizerInfo( encoded_vocab, vocab_type=vocab_type, vocab_size=vocab_size, stop_token_ids=stop_token_ids, add_prefix_space=True, ) else: # TODO(yixin): unsupported tokenizer raise ValueError(f"Unsupported tokenizer type: {type(tokenizer)}") @property def vocab_type(self) -> VocabType: """The type of the vocabulary.""" return VocabType(self._handle.vocab_type()) @property def vocab_size(self) -> int: """The size of the vocabulary.""" return self._handle.vocab_size() @property def add_prefix_space(self) -> bool: """Whether the tokenizer will prepend a space before the text in the tokenization process.""" return self._handle.add_prefix_space() @property def prepend_space_in_tokenization(self) -> bool: """Whether the tokenizer will prepend a space before the text in the tokenization process. This property is deprecated. Use add_prefix_space instead. """ warnings.warn("prepend_space_in_tokenization is deprecated. Use add_prefix_space instead.") return self.add_prefix_space @property def decoded_vocab(self) -> List[bytes]: """The decoded vocabulary of the tokenizer. This converts the tokens in the LLM's vocabulary back to the original format of the input text. E.g. for type ByteFallback, the token <0x1B> is converted back to "\u001b". """ return list(self._handle.decoded_vocab()) @property def stop_token_ids(self) -> List[int]: """The stop token ids.""" return list(self._handle.stop_token_ids()) @property def special_token_ids(self) -> List[int]: """The special token ids. Special tokens include control tokens, reserved tokens, padded tokens, etc. Now it is automatically detected from the vocabulary.""" return list(self._handle.special_token_ids()) def dump_metadata(self) -> str: """Dump the metadata of the tokenizer to a json string. It can be used to construct the tokenizer info from the vocabulary and the metadata string.""" return str(self._handle.dump_metadata()) @staticmethod def from_vocab_and_metadata( encoded_vocab: List[Union[bytes, str]], metadata: str ) -> "TokenizerInfo": """Construct the tokenizer info from the vocabulary and the metadata string in json format. Parameters ---------- encoded_vocab : List[Union[bytes, str]] The encoded vocabulary of the tokenizer. metadata : str The metadata string in json format. """ return TokenizerInfo._create_from_handle( _core.TokenizerInfo.from_vocab_and_metadata(encoded_vocab, metadata) ) @staticmethod def _detect_metadata_from_hf(backend_str: str) -> Dict[str, Any]: """Detect the metadata from the huggingface tokenizer backend string. For implementation use only. It returns {"vocab_type": VocabType, "add_prefix_space": bool}. """ # the metadata_str should in the format of {"vocab_type": int, "add_prefix_space": bool} metadata_str = _core.TokenizerInfo._detect_metadata_from_hf(backend_str) metadata = json.loads(metadata_str) return { "vocab_type": VocabType(metadata["vocab_type"]), "add_prefix_space": metadata["add_prefix_space"], } def serialize_json(self) -> str: """Serialize the tokenizer info to a JSON string. Returns ------- json_string : str The JSON string. """ return str(self._handle.serialize_json()) @staticmethod def deserialize_json(json_string: str) -> "TokenizerInfo": """Deserialize a tokenizer info from a JSON string. Parameters ---------- json_string : str The JSON string. Returns ------- tokenizer_info : TokenizerInfo The tokenizer info. Raises ------ InvalidJSONError When the JSON string is invalid. DeserializeFormatError When the JSON string does not follow the serialization format of the tokenizer info. DeserializeVersionError When the __VERSION__ field in the JSON string is not the same as the current version. """ return TokenizerInfo._create_from_handle(_core.TokenizerInfo.deserialize_json(json_string)) xgrammar-0.2.3/scripts/000077500000000000000000000000001521764210300150225ustar00rootroot00000000000000xgrammar-0.2.3/scripts/build-environment.yaml000066400000000000000000000003571521764210300213540ustar00rootroot00000000000000name: xgrammar-build channels: - pytorch - conda-forge dependencies: - llvmdev>=15 - cmake>=3.24 - zlib - zstd-static - git - conda-build - numpy - pytest - pip - cython - apache-tvm-ffi - pytorch - cpuonly xgrammar-0.2.3/scripts/docker/000077500000000000000000000000001521764210300162715ustar00rootroot00000000000000xgrammar-0.2.3/scripts/docker/bash.sh000077500000000000000000000044341521764210300175520ustar00rootroot00000000000000#!/usr/bin/env bash # # Start a bash, mount /workspace to be current directory. # # Usage: docker/bash.sh # Starts an interactive session # # Usage2: docker/bash.sh [COMMAND] # Execute command in the docker image, non-interactive # if [ "$#" -lt 1 ]; then echo "Usage: docker/bash.sh [--no-gpu] [COMMAND]" exit -1 fi if [ "$1" == "--no-gpu" ]; then ENABLE_NV_DOCKER=0 shift else ENABLE_NV_DOCKER=1 fi DOCKER_IMAGE_NAME=("$1") if [ "$#" -eq 1 ]; then COMMAND="bash" if [[ $(uname) == "Darwin" ]]; then # Docker's host networking driver isn't supported on macOS. # Use default bridge network and expose port for jupyter notebook. DOCKER_EXTRA_PARAMS=("-it -p 8888:8888") else DOCKER_EXTRA_PARAMS=("-it --net=host") fi else shift 1 COMMAND=("$@") fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" WORKSPACE="$(pwd)" # Use nvidia-docker if the container is GPU. if [[ ! -z $CUDA_VISIBLE_DEVICES ]]; then CUDA_ENV="-e CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES}" else CUDA_ENV="" fi # If this is an wheel test command then pass the env var to docker. if [[ ! -z $WHEEL_TEST ]]; then WHEEL_TEST="-e WHEEL_TEST=${WHEEL_TEST}" fi if [[ "${DOCKER_IMAGE_NAME}" == *"cu"* ]]; then if [ "$ENABLE_NV_DOCKER" -eq 1 ]; then if ! type "nvidia-docker" 1> /dev/null 2> /dev/null then DOCKER_BINARY="docker" CUDA_ENV=" --gpus all "${CUDA_ENV} else DOCKER_BINARY="nvidia-docker" fi else DOCKER_BINARY="docker" fi else DOCKER_BINARY="docker" fi # Print arguments. echo "WORKSPACE: ${WORKSPACE}" echo "DOCKER CONTAINER NAME: ${DOCKER_IMAGE_NAME}" echo "" echo "Running '${COMMAND[@]}' inside ${DOCKER_IMAGE_NAME}..." # By default we cleanup - remove the container once it finish running (--rm) # and share the PID namespace (--pid=host) so the process inside does not have # pid 1 and SIGKILL is propagated to the process inside (jenkins can kill it). ${DOCKER_BINARY} run --rm --pid=host\ -v ${WORKSPACE}:/workspace \ -v ${SCRIPT_DIR}:/docker \ -w /workspace \ ${CUDA_ENV} \ ${WHEEL_TEST} \ ${DOCKER_EXTRA_PARAMS[@]} \ ${DOCKER_IMAGE_NAME} \ ${COMMAND[@]} xgrammar-0.2.3/scripts/gh_deploy_site.sh000077500000000000000000000010011521764210300203470ustar00rootroot00000000000000#!/bin/bash # Build the docs and the site, then deploy to github pages # NOTE: this script is triggered by github action automatically # when megred into main set -euxo pipefail scripts/support/build_site.sh git fetch git checkout -B gh-pages origin/gh-pages rm -rf docs .gitignore mkdir -p docs cp -rf site/_site/* docs touch docs/.nojekyll DATE=`date` git add docs && git commit -am "Build at ${DATE}" git push origin gh-pages git checkout main && git submodule update echo "Finish deployment at ${DATE}" xgrammar-0.2.3/scripts/lint.sh000077500000000000000000000001221521764210300163220ustar00rootroot00000000000000#!/usr/bin/env bash set -e set -x pre-commit run --all-files ruff check . --fix xgrammar-0.2.3/scripts/local_deploy_site.sh000077500000000000000000000003321521764210300210510ustar00rootroot00000000000000#!/bin/bash # Build the docs and the site, then serve the site locally set -euxo pipefail scripts/support/build_site.sh cd site && jekyll serve --trace --skip-initial-build --host localhost --baseurl / --port 8888 xgrammar-0.2.3/scripts/release_new_version.sh000077500000000000000000000004471521764210300214240ustar00rootroot00000000000000#!/bin/bash # Usage: ./scripts/release_new_version.sh set -ex if [ -z "$1" ]; then echo "Error: Version argument is required" echo "Usage: $0 " exit 1 fi # Pull and checkout main branch git pull origin main git checkout main git tag $1 HEAD git push origin $1 xgrammar-0.2.3/scripts/run_coverage.sh000077500000000000000000000006651521764210300200470ustar00rootroot00000000000000#!/bin/bash # Usage: bash ./scripts/run_coverage.sh lcov --directory . --zerocounters ctest --test-dir build -V --timeout 60 --stop-on-failure pytest lcov --gcov-tool /usr/bin/gcov-13 --directory . --capture --output-file coverage.info --ignore-errors mismatch,gcov genhtml coverage.info --output-directory coverage_report --ignore-errors version rm coverage.info echo "Coverage report generated at: $(pwd)/coverage_report/index.html" xgrammar-0.2.3/scripts/run_ctest.sh000077500000000000000000000005711521764210300173720ustar00rootroot00000000000000#!/usr/bin/env bash # Usage: # ./scripts/run_ctest.sh [name_of_test_to_run] set -euxo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/../build" && cmake .. -G Ninja && ninja if [ $# -gt 0 ]; then # If a test name is given, run only that test ctest -R "$1" --verbose --timeout 30 else # If no argument is given, run the full test suite ctest --verbose --timeout 30 fi xgrammar-0.2.3/scripts/support/000077500000000000000000000000001521764210300165365ustar00rootroot00000000000000xgrammar-0.2.3/scripts/support/build_site.sh000077500000000000000000000003551521764210300212230ustar00rootroot00000000000000#!/bin/bash # build the docs and the site to site/_site set -euxo pipefail export PYTHONPATH=$PWD/python cd docs && ./build_docs.sh && cd .. cd site && jekyll b && cd .. rm -rf site/_site/docs cp -r docs/_build/html site/_site/docs xgrammar-0.2.3/site/000077500000000000000000000000001521764210300142775ustar00rootroot00000000000000xgrammar-0.2.3/site/.gitignore000066400000000000000000000000311521764210300162610ustar00rootroot00000000000000dist _site .jekyll-cache xgrammar-0.2.3/site/CNAME000066400000000000000000000000201521764210300150350ustar00rootroot00000000000000xgrammar.mlc.ai xgrammar-0.2.3/site/Gemfile000066400000000000000000000002021521764210300155640ustar00rootroot00000000000000# frozen_string_literal: true source "https://rubygems.org" # gem "rails" gem "jekyll-remote-theme" gem "jekyll-sass-converter" xgrammar-0.2.3/site/_config.yml000066400000000000000000000013431521764210300164270ustar00rootroot00000000000000name: "XGrammar" short_name: "XGrammar" url: https://xgrammar.mlc.ai/ exclude: [README.md, serve_local.sh] plugins: - jekyll-remote-theme remote_theme: mlc-ai/jekyll-theme-mlc # Colorize code snippets with the rogue module if we want to deploy on GH. highlighter: rouge markdown: kramdown # The path structure for blog posts. permalink: /blog/:year/:month/:day/:title.html # Number of news stories on the front page. front_page_news: 8 # Base pathname for links. base: '' # make pages for the _projects folder collections: projects: output: true course_title: # Navigation bar links. navigation: - title: Home link: / - title: Docs link: /docs - title: Github link: https://github.com/mlc-ai/xgrammar xgrammar-0.2.3/site/_includes/000077500000000000000000000000001521764210300162445ustar00rootroot00000000000000xgrammar-0.2.3/site/_includes/arrow.svg000066400000000000000000000015201521764210300201150ustar00rootroot00000000000000 xgrammar-0.2.3/site/_includes/github.svg000066400000000000000000000017301521764210300202500ustar00rootroot00000000000000 xgrammar-0.2.3/site/_includes/head.html000066400000000000000000000014761521764210300200430ustar00rootroot00000000000000 xgrammar-0.2.3/site/_includes/hero.html000066400000000000000000000020671521764210300200740ustar00rootroot00000000000000

XGrammar: Efficient, Flexible and Portable Structured Generation

{% include github.svg %} GitHub {% include arrow.svg %} Documentation {% include arrow.svg %}
xgrammar-0.2.3/site/assets/000077500000000000000000000000001521764210300156015ustar00rootroot00000000000000xgrammar-0.2.3/site/assets/css/000077500000000000000000000000001521764210300163715ustar00rootroot00000000000000xgrammar-0.2.3/site/assets/css/hero.scss000066400000000000000000000125661521764210300202350ustar00rootroot00000000000000--- --- #hero { background: radial-gradient(100% 50rem at center 50rem, #3351cb50, #ffffff); padding: 3rem; width: 100vw; margin-left: calc(50% - 50vw); margin-top: -20px; display: flex; flex-direction: column; align-items: center; a { color: black; } .heading-container { display: flex; flex-direction: column; align-items: center; font-family: "Mona Sans", "MonaSansFallback", -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji"; margin: auto; a { min-width: fit-content; max-width: 16rem; flex-grow: 1; } h1 { text-align: center; font-size: 2rem; font-weight: 700; } .link-container { display: flex; margin-top: 2rem; align-items: center; flex-wrap: wrap; font-size: 1rem; word-break: keep-all; font-weight: 600; gap: 1rem; justify-content: center; .github-link { display: inline-flex; gap: 1rem; border-radius: 9999px; vertical-align: middle; align-items: center; justify-content: center; text-decoration: none; cursor: pointer; height: fit-content; // padding: .25rem; .github-link-content { width: 100%; height: 100%; z-index: 1; border-radius: 9999px; padding: 1rem 1.75rem; background-color: #000000; display: inline-flex; gap: .5rem; display: inline-flex; justify-content: center; color: rgb(229 229 229); .icon { display: inline-flex; align-items: center; margin-right: .5rem; svg { height: 1.5rem; } } } } .get-start-link { display: inline-flex; gap: 1rem; background-color: white; border-radius: 9999px; vertical-align: middle; align-items: center; justify-content: center; text-decoration: none; cursor: pointer; height: fit-content; padding: .25rem; .get-start-link-content { width: 100%; height: 100%; z-index: 1; border-radius: 9999px; padding: 1rem 1.75rem; background-color: white; display: inline-flex; justify-content: center; } } .arrow-container { margin-left: .25rem; display: inline-flex; align-items: center; } } } .arrow-expandable { stroke-dasharray: 10; stroke-dashoffset: 10; transition: stroke-dashoffset 200ms; } .expanded { .arrow-expandable { stroke-dashoffset: 20; } } .demo-container { position: relative; margin-top: 96px; width: calc(100% + 4rem); max-width: 1024px; flex-shrink: 0; padding: 2rem; svg { height: auto; width: 100%; border-radius: inherit; } } } .moving-border { overflow: hidden; position: relative; .border { position: absolute; inset: -1000%; animation: spin 3s linear infinite; border-radius: 1rem; background-image: conic-gradient(from 90deg at 50% 50%, #e2cbff 0, #393bb2 50%, #e2cbff 100%); } } @media screen and (min-width:640px) { #hero { padding: 6rem; .heading-container { max-width: 40rem; h1 { font-size: 3rem; } } .demo-container { width: calc(100% + 10rem); } } } @media screen and (min-width:768px) { #hero { .heading-container { max-width: 45rem; h1 { font-size: 3.2rem; } .link-container { font-size: 1.2rem; } } } } @media screen and (min-width:1024px) { #hero { padding: 8rem; .heading-container { max-width: 50rem; h1 { font-size: 3.5rem; } } .demo-container { width: 100%; } } } @media screen and (min-width:1280px) { #hero { .heading-container { max-width: 60rem; h1 { font-size: 4rem; } } } } @media screen and (min-width:1760px) { #hero { background: radial-gradient(100% 50rem at center 50rem, #3351cb50, #ffffff); gap: 4rem; padding-bottom: 12rem; } } @keyframes spin { 100% { transform: rotate(1turn); } } xgrammar-0.2.3/site/index.md000066400000000000000000000016651521764210300157400ustar00rootroot00000000000000--- layout: default title: Home notitle: true --- {% include hero.html %} ## Overview XGrammar is open-source solution for flexible, portable, and fast structured generations, aiming at bring flexible zero-overhead structure generation everywhere. It supports general context-free grammar to enable a broad range of structures while bringing careful system optimizations to enable fast executions. XGrammar features a minimal and portable C++ backend that can be easily integrated into multiple environments and frameworks, and is co-designed with the LLM inference engine and enables zero-overhead structured generation in LLM inference. ## Get Started Please visit our [documentation](https://xgrammar.mlc.ai/docs/) to get started with XGrammar. - [Installation](https://xgrammar.mlc.ai/docs/start/installation) - [Quick start](https://xgrammar.mlc.ai/docs/start/quick_start) ## Links - [XGrammar Github](https://github.com/mlc-ai/xgrammar) xgrammar-0.2.3/tests/000077500000000000000000000000001521764210300144755ustar00rootroot00000000000000xgrammar-0.2.3/tests/README.md000066400000000000000000000005571521764210300157630ustar00rootroot00000000000000To test, run `pytest .` under `xgrammar` folder. You may need to do the following: ```bash pip install sentencepiece pip install protobuf pip install -U "huggingface_hub[cli]" huggingface-cli login --token YOUR_HF_TOKEN ``` Make sure you also have access to the gated models, which should only require you to agree some terms on the models' website on huggingface. xgrammar-0.2.3/tests/conftest.py000066400000000000000000000026071521764210300167010ustar00rootroot00000000000000import os from pathlib import Path import pytest try: import pytest_run_parallel # noqa: F401 PARALLEL_RUN_AVAILABLE = True except ModuleNotFoundError: PARALLEL_RUN_AVAILABLE = False def _hf_token_available() -> bool: """Check whether a HuggingFace token is available via env vars or cached login.""" if os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN"): return True # huggingface-cli login stores token here return Path.home().joinpath(".cache", "huggingface", "token").is_file() def _hf_token_explicitly_disabled(config) -> bool: """Return whether pytest mark expression explicitly excludes hf-token tests.""" markexpr = config.getoption("markexpr", "") return "not hf_token_required" in markexpr def pytest_configure(config): if not PARALLEL_RUN_AVAILABLE: config.addinivalue_line( "markers", "thread_unsafe: mark the test function as single-threaded" ) def pytest_collection_modifyitems(config, items): if _hf_token_available(): return skip_no_token = pytest.mark.skip( reason="HF_TOKEN not set (run `huggingface-cli login` or set HF_TOKEN env var)" ) for item in items: if "hf_token_required" in item.keywords: item.add_marker(skip_no_token) if not PARALLEL_RUN_AVAILABLE: @pytest.fixture def num_parallel_threads(): return 1 xgrammar-0.2.3/tests/cpp/000077500000000000000000000000001521764210300152575ustar00rootroot00000000000000xgrammar-0.2.3/tests/cpp/test_fsm.cc000066400000000000000000000643071521764210300174240ustar00rootroot00000000000000/** * \file tests/cpp/test_fsm.cc * \brief Test FSM operations. */ #include #include #include #include #include "fsm.h" #include "fsm_builder.h" #include "support/logging.h" using namespace xgrammar; TEST(XGrammarFSMTest, BasicBuildTest) { std::cout << "--------- Basic Build Test Starts! -----------" << std::endl; std::cout << "--------- Basic Build Test1 -----------" << std::endl; auto fsm_wse = RegexFSMBuilder::Build("abcd\\n").Unwrap(); std::string test_str = "abcd\n"; EXPECT_TRUE(fsm_wse.AcceptString(test_str)); std::cout << "--------- Basic Build Test2 -----------" << std::endl; fsm_wse = RegexFSMBuilder::Build("[-a-z\\n]").Unwrap(); test_str = "abcd-\n"; for (const auto& character : test_str) { EXPECT_TRUE([&]() -> bool { for (const auto& edge : fsm_wse.GetFsm().GetEdges(0)) { if (edge.min <= int(character) && edge.max >= int(character)) { return true; } } return false; }()); } std::cout << "--------- Basic Build Test3 -----------" << std::endl; fsm_wse = RegexFSMBuilder::Build("[\\d]").Unwrap(); test_str = "1234567890"; for (const auto& character : test_str) { EXPECT_TRUE([&]() -> bool { for (const auto& edge : fsm_wse.GetFsm().GetEdges(0)) { if (edge.min <= int(character) && edge.max >= int(character)) { return true; } } return false; }()); } std::cout << "--------- Basic Build Test4 -----------" << std::endl; fsm_wse = RegexFSMBuilder::Build("[^\\d]").Unwrap(); test_str = "1234567890"; for (const auto& character : test_str) { EXPECT_TRUE([&]() -> bool { for (const auto& edge : fsm_wse.GetFsm().GetEdges(0)) { if (edge.min <= int(character) && edge.max >= int(character)) { return false; } } return true; }()); } test_str = "abz"; for (const auto& character : test_str) { EXPECT_TRUE([&]() -> bool { for (const auto& edge : fsm_wse.GetFsm().GetEdges(0)) { if (edge.min <= int(character) && edge.max >= int(character)) { return true; } } std::cout << character << std::endl; return false; }()); } std::cout << "--------- Basic Build Test5 -----------" << std::endl; fsm_wse = RegexFSMBuilder::Build("你好a").Unwrap(); test_str = "你好a"; EXPECT_TRUE(fsm_wse.AcceptString(test_str)); std::cout << "--------- Basic Build Test6 -----------" << std::endl; fsm_wse = RegexFSMBuilder::Build("(())()()").Unwrap(); test_str = ""; EXPECT_TRUE(fsm_wse.AcceptString(test_str)); std::cout << "--------- Basic Build Test7 -----------" << std::endl; fsm_wse = RegexFSMBuilder::Build("[abcdabcdxyzxyz]").Unwrap(); test_str = "a"; std::cout << fsm_wse << std::endl; EXPECT_TRUE(fsm_wse.AcceptString(test_str)); EXPECT_FALSE(fsm_wse.AcceptString("e")); std::cout << fsm_wse << std::endl; EXPECT_EQ(fsm_wse.GetFsm().GetEdges(0).size(), 2); std::cout << "Basic Build Test Passed!" << std::endl; } TEST(XGrammarFSMTest, ConnectionTest) { std::cout << "--------- Connection Test Starts! -----------" << std::endl; std::cout << "--------- Connection Test1 -----------" << std::endl; auto fsm_wse = RegexFSMBuilder::Build(" [a-zA-Z0-9]--").Unwrap(); std::string test_str = " a--"; EXPECT_TRUE(fsm_wse.AcceptString(test_str)); std::cout << "--------- Connection Test2 -----------" << std::endl; fsm_wse = RegexFSMBuilder::Build("aaa|[\\d]").Unwrap(); test_str = "aaa"; EXPECT_TRUE(fsm_wse.AcceptString(test_str)); test_str = "1"; EXPECT_TRUE(fsm_wse.AcceptString(test_str)); std::cout << "--------- Connection Test3 -----------" << std::endl; auto result = RegexFSMBuilder::Build("(([\\d]|[\\w])|aaa)"); EXPECT_FALSE(result.IsErr()) << std::move(result).UnwrapErr().what(); fsm_wse = std::move(result).Unwrap(); test_str = "aaa"; EXPECT_TRUE(fsm_wse.AcceptString(test_str)); test_str = "1"; EXPECT_TRUE(fsm_wse.AcceptString(test_str)); test_str = "1a"; EXPECT_FALSE(fsm_wse.AcceptString(test_str)); std::cout << "Connection Test Passed!" << std::endl; } TEST(XGrammarFSMTest, SymbolTest) { std::cout << "--------- Symbol Test Starts! -----------" << std::endl; std::cout << "--------- Symbol Test1 -----------" << std::endl; auto fsm_wse = RegexFSMBuilder::Build("1[\\d]+").Unwrap(); std::string test_str[2] = {"1111", "1"}; EXPECT_TRUE(fsm_wse.AcceptString(test_str[0])); EXPECT_FALSE(fsm_wse.AcceptString(test_str[1])); std::cout << "--------- Symbol Test2 -----------" << std::endl; fsm_wse = RegexFSMBuilder::Build("1[1]*").Unwrap(); EXPECT_TRUE(fsm_wse.AcceptString(test_str[0])); EXPECT_TRUE(fsm_wse.AcceptString(test_str[1])); std::cout << "--------- Symbol Test3 -----------" << std::endl; fsm_wse = RegexFSMBuilder::Build("1[\\d]?").Unwrap(); EXPECT_FALSE(fsm_wse.AcceptString(test_str[0])); EXPECT_TRUE(fsm_wse.AcceptString(test_str[1])); std::string test3 = "11"; EXPECT_TRUE(fsm_wse.AcceptString(test3)); std::cout << "--------- Symbol Test4 -----------" << std::endl; fsm_wse = RegexFSMBuilder::Build(" * * + ? *").Unwrap(); test_str[0] = " "; test_str[1] = " "; for (const auto& str : test_str) { EXPECT_TRUE(fsm_wse.AcceptString(str)); } std::cout << "Symbol Test Passed!" << std::endl; } TEST(XGrammarFSMTest, IntegratedTest) { std::cout << "--------- Integrated Test Starts! -----------" << std::endl; auto fsm_wse = RegexFSMBuilder::Build("((naive|bbb|[\\d]+)*[\\w])| +").Unwrap(); std::string test_str[5] = {"naive1", "bbbnaive114514W", " ", "123", "_"}; for (const auto& str : test_str) { EXPECT_TRUE(fsm_wse.AcceptString(str)); } std::string test_str2[5] = {"naive", "bbbbbb", "naive ", "123 ", "aaa"}; for (const auto& str : test_str2) { EXPECT_FALSE(fsm_wse.AcceptString(str)); } std::cout << "--------- Integrated Test Passed! -----------" << std::endl; } TEST(XGrammarFSMTest, FunctionTest) { std::cout << "--------- Function Test Starts! -----------" << std::endl; std::cout << "--------- Function Test1 -----------" << std::endl; auto fsm_wse = RegexFSMBuilder::Build("[\\d\\d\\d]+123").Unwrap(); std::string test_str = "123456123"; EXPECT_TRUE(fsm_wse.AcceptString(test_str)); auto compact_fsm = fsm_wse.GetFsm().ToCompact(); CompactFSMWithStartEnd compact_fsm_wse(compact_fsm, fsm_wse.GetStart(), fsm_wse.GetEnds()); EXPECT_TRUE(compact_fsm_wse.AcceptString(test_str)); fsm_wse = FSMWithStartEnd(compact_fsm.ToFSM(), fsm_wse.GetStart(), fsm_wse.GetEnds()); EXPECT_TRUE(fsm_wse.AcceptString(test_str)); std::cout << "--------- Function Test2 -----------" << std::endl; fsm_wse = RegexFSMBuilder::Build("([abc]|[\\d])+").Unwrap(); test_str = "abc3"; EXPECT_TRUE(fsm_wse.AcceptString(test_str)); fsm_wse = std::move(fsm_wse.ToDFA()).Unwrap(); EXPECT_TRUE(fsm_wse.AcceptString(test_str)); EXPECT_TRUE([&]() -> bool { for (const auto& edges : fsm_wse.GetFsm().GetEdges()) { for (const auto& edge : edges) { if (edge.IsEpsilon()) { return false; } } } return true; }()); EXPECT_TRUE([&]() -> bool { for (const auto& edges : fsm_wse.GetFsm().GetEdges()) { std::unordered_set rules; std::unordered_set chars; for (const auto& edge : edges) { if (edge.IsRuleRef()) { if (rules.find(edge.GetRefRuleId()) != rules.end()) { return false; } rules.insert(edge.GetRefRuleId()); continue; } for (int i = edge.min; i <= edge.max; i++) { if (chars.find(i) != chars.end()) { return false; } chars.insert(i); } } } return true; }()); std::cout << "--------- Function Test3 -----------" << std::endl; fsm_wse = std::move(fsm_wse.MinimizeDFA()).Unwrap(); EXPECT_TRUE(fsm_wse.AcceptString(test_str)); EXPECT_EQ(fsm_wse.GetFsm().GetEdges().size(), 2); std::cout << "--------- Function Test4 -----------" << std::endl; fsm_wse = std::move(fsm_wse.Not()).Unwrap(); EXPECT_FALSE(fsm_wse.AcceptString(test_str)); test_str = "abcd"; EXPECT_TRUE(fsm_wse.AcceptString(test_str)); std::cout << "--------- Function Test5 -----------" << std::endl; fsm_wse = RegexFSMBuilder::Build("[\\d]{1,5}").Unwrap(); std::string test_strs[2] = {"123", "12345"}; for (const auto& str : test_strs) { EXPECT_TRUE(fsm_wse.AcceptString(str)); } test_strs[0] = "123456"; test_strs[1] = "1234567"; for (const auto& str : test_strs) { EXPECT_FALSE(fsm_wse.AcceptString(str)); } fsm_wse = RegexFSMBuilder::Build("[\\d]{6}").Unwrap(); EXPECT_TRUE(fsm_wse.AcceptString("123456")); EXPECT_FALSE(fsm_wse.AcceptString("1234567")); fsm_wse = RegexFSMBuilder::Build("[\\d]{6, }").Unwrap(); EXPECT_TRUE(fsm_wse.AcceptString("123456")); EXPECT_TRUE(fsm_wse.AcceptString("1234567")); std::cout << "--------- Function Test6 -----------" << std::endl; fsm_wse = RegexFSMBuilder::Build("[a][b][c][d]").Unwrap(); test_str = "abcd"; EXPECT_TRUE(fsm_wse.AcceptString(test_str)); fsm_wse = fsm_wse.SimplifyEpsilon(); std::cout << fsm_wse << std::endl; EXPECT_EQ(fsm_wse.GetFsm().NumStates(), 5); EXPECT_TRUE(fsm_wse.AcceptString(test_str)); std::cout << "--------- Function Test7 -----------" << std::endl; fsm_wse = RegexFSMBuilder::Build("abc|abd").Unwrap(); test_str = "abc"; EXPECT_TRUE(fsm_wse.AcceptString(test_str)); fsm_wse = fsm_wse.SimplifyEpsilon(); fsm_wse = fsm_wse.MergeEquivalentStates(); EXPECT_TRUE(fsm_wse.AcceptString(test_str)); test_str = "abcd"; EXPECT_FALSE(fsm_wse.AcceptString(test_str)); EXPECT_EQ(fsm_wse.GetFsm().NumStates(), 4); std::cout << "--------- Function Test8 -----------" << std::endl; fsm_wse = RegexFSMBuilder::Build("acd|bcd").Unwrap(); test_str = "acd"; EXPECT_TRUE(fsm_wse.AcceptString(test_str)); fsm_wse = fsm_wse.SimplifyEpsilon(); fsm_wse = fsm_wse.MergeEquivalentStates(); EXPECT_TRUE(fsm_wse.AcceptString(test_str)); test_str = "abcd"; EXPECT_FALSE(fsm_wse.AcceptString(test_str)); EXPECT_EQ(fsm_wse.GetFsm().NumStates(), 4); XGRAMMAR_LOG(INFO) << fsm_wse; std::cout << "--------- Function Test9 -----------" << std::endl; fsm_wse = RegexFSMBuilder::Build("ab*").Unwrap(); test_str = "abbb"; EXPECT_TRUE(fsm_wse.AcceptString(test_str)); fsm_wse = fsm_wse.SimplifyEpsilon(); EXPECT_TRUE(fsm_wse.AcceptString(test_str)); EXPECT_EQ(fsm_wse.GetFsm().NumStates(), 2); std::cout << "--------- Function Test10 -----------" << std::endl; const auto fsm_left = RegexFSMBuilder::Build("[c-f]+").Unwrap(); const auto fsm_right = RegexFSMBuilder::Build("[d-h]*").Unwrap(); std::cout << fsm_left << std::endl; std::cout << fsm_right << std::endl; fsm_wse = FSMWithStartEnd::Intersect(fsm_left, fsm_right).Unwrap(); std::cout << fsm_wse << std::endl; EXPECT_TRUE(fsm_wse.AcceptString("de")); EXPECT_TRUE(fsm_wse.AcceptString("def")); EXPECT_FALSE(fsm_wse.AcceptString("")); EXPECT_FALSE(fsm_wse.AcceptString("cd")); std::cout << "--------- Function Test Passed! -----------" << std::endl; } TEST(XGrammarFSMTest, EfficiencyTest) { std::cout << "--------- Efficiency Test Starts! -----------" << std::endl; // i.e ([a-z]0123456789){10}. Use this way to test the performance. auto fsm_wse = RegexFSMBuilder::Build( "(a0123456789|a0123456789|b0123456789|b0123456789|c0123456789|" "c0123456789|d0123456789|d0123456789|e0123456789|e0123456789|" "f0123456789|f0123456789|g0123456789|g0123456789|h0123456789|" "h0123456789|i0123456789|i0123456789|j0123456789|j0123456789|" "k0123456789|k0123456789|l0123456789|l0123456789|m0123456789|" "m0123456789|n0123456789|n0123456789|o0123456789|o0123456789|" "p0123456789|p0123456789|q0123456789|q0123456789|r0123456789|" "r0123456789|s0123456789|s0123456789|t0123456789|t0123456789|" "u0123456789|u0123456789|v0123456789|v0123456789|w0123456789|" "w0123456789|x0123456789|x0123456789|y0123456789|y0123456789|" "z0123456789|z0123456789)(a0123456789|a0123456789|b0123456789|" "b0123456789|c0123456789|c0123456789|d0123456789|d0123456789|" "e0123456789|e0123456789|f0123456789|f0123456789|g0123456789|" "g0123456789|h0123456789|h0123456789|i0123456789|i0123456789|" "j0123456789|j0123456789|k0123456789|k0123456789|l0123456789|" "l0123456789|m0123456789|m0123456789|n0123456789|n0123456789|" "o0123456789|o0123456789|p0123456789|p0123456789|q0123456789|" "q0123456789|r0123456789|r0123456789|s0123456789|s0123456789|" "t0123456789|t0123456789|u0123456789|u0123456789|v0123456789|" "v0123456789|w0123456789|w0123456789|x0123456789|x0123456789|" "y0123456789|y0123456789|z0123456789|z0123456789)(a0123456789|" "a0123456789|b0123456789|b0123456789|c0123456789|c0123456789|" "d0123456789|d0123456789|e0123456789|e0123456789|f0123456789|" "f0123456789|g0123456789|g0123456789|h0123456789|h0123456789|" "i0123456789|i0123456789|j0123456789|j0123456789|k0123456789|" "k0123456789|l0123456789|l0123456789|m0123456789|m0123456789|" "n0123456789|n0123456789|o0123456789|o0123456789|p0123456789|" "p0123456789|q0123456789|q0123456789|r0123456789|r0123456789|" "s0123456789|s0123456789|t0123456789|t0123456789|u0123456789|" "u0123456789|v0123456789|v0123456789|w0123456789|w0123456789|" "x0123456789|x0123456789|y0123456789|y0123456789|z0123456789|" "z0123456789)(a0123456789|a0123456789|b0123456789|b0123456789|" "c0123456789|c0123456789|d0123456789|d0123456789|e0123456789|" "e0123456789|f0123456789|f0123456789|g0123456789|g0123456789|" "h0123456789|h0123456789|i0123456789|i0123456789|j0123456789|" "j0123456789|k0123456789|k0123456789|l0123456789|l0123456789|" "m0123456789|m0123456789|n0123456789|n0123456789|o0123456789|" "o0123456789|p0123456789|p0123456789|q0123456789|q0123456789|" "r0123456789|r0123456789|s0123456789|s0123456789|t0123456789|" "t0123456789|u0123456789|u0123456789|v0123456789|v0123456789|" "w0123456789|w0123456789|x0123456789|x0123456789|y0123456789|" "y0123456789|z0123456789|z0123456789)(a0123456789|a0123456789|" "b0123456789|b0123456789|c0123456789|c0123456789|d0123456789|" "d0123456789|e0123456789|e0123456789|f0123456789|f0123456789|" "g0123456789|g0123456789|h0123456789|h0123456789|i0123456789|" "i0123456789|j0123456789|j0123456789|k0123456789|k0123456789|" "l0123456789|l0123456789|m0123456789|m0123456789|n0123456789|" "n0123456789|o0123456789|o0123456789|p0123456789|p0123456789|" "q0123456789|q0123456789|r0123456789|r0123456789|s0123456789|" "s0123456789|t0123456789|t0123456789|u0123456789|u0123456789|" "v0123456789|v0123456789|w0123456789|w0123456789|x0123456789|" "x0123456789|y0123456789|y0123456789|z0123456789|z0123456789)(" "a0123456789|a0123456789|b0123456789|b0123456789|c0123456789|" "c0123456789|d0123456789|d0123456789|e0123456789|e0123456789|" "f0123456789|f0123456789|g0123456789|g0123456789|h0123456789|" "h0123456789|i0123456789|i0123456789|j0123456789|j0123456789|" "k0123456789|k0123456789|l0123456789|l0123456789|m0123456789|" "m0123456789|n0123456789|n0123456789|o0123456789|o0123456789|" "p0123456789|p0123456789|q0123456789|q0123456789|r0123456789|" "r0123456789|s0123456789|s0123456789|t0123456789|t0123456789|" "u0123456789|u0123456789|v0123456789|v0123456789|w0123456789|" "w0123456789|x0123456789|x0123456789|y0123456789|y0123456789|" "z0123456789|z0123456789)(a0123456789|a0123456789|b0123456789|" "b0123456789|c0123456789|c0123456789|d0123456789|d0123456789|" "e0123456789|e0123456789|f0123456789|f0123456789|g0123456789|" "g0123456789|h0123456789|h0123456789|i0123456789|i0123456789|" "j0123456789|j0123456789|k0123456789|k0123456789|l0123456789|" "l0123456789|m0123456789|m0123456789|n0123456789|n0123456789|" "o0123456789|o0123456789|p0123456789|p0123456789|q0123456789|" "q0123456789|r0123456789|r0123456789|s0123456789|s0123456789|" "t0123456789|t0123456789|u0123456789|u0123456789|v0123456789|" "v0123456789|w0123456789|w0123456789|x0123456789|x0123456789|" "y0123456789|y0123456789|z0123456789|z0123456789)(a0123456789|" "a0123456789|b0123456789|b0123456789|c0123456789|c0123456789|" "d0123456789|d0123456789|e0123456789|e0123456789|f0123456789|" "f0123456789|g0123456789|g0123456789|h0123456789|h0123456789|" "i0123456789|i0123456789|j0123456789|j0123456789|k0123456789|" "k0123456789|l0123456789|l0123456789|m0123456789|m0123456789|" "n0123456789|n0123456789|o0123456789|o0123456789|p0123456789|" "p0123456789|q0123456789|q0123456789|r0123456789|r0123456789|" "s0123456789|s0123456789|t0123456789|t0123456789|u0123456789|" "u0123456789|v0123456789|v0123456789|w0123456789|w0123456789|" "x0123456789|x0123456789|y0123456789|y0123456789|z0123456789|" "z0123456789)(a0123456789|a0123456789|b0123456789|b0123456789|" "c0123456789|c0123456789|d0123456789|d0123456789|e0123456789|" "e0123456789|f0123456789|f0123456789|g0123456789|g0123456789|" "h0123456789|h0123456789|i0123456789|i0123456789|j0123456789|" "j0123456789|k0123456789|k0123456789|l0123456789|l0123456789|" "m0123456789|m0123456789|n0123456789|n0123456789|o0123456789|" "o0123456789|p0123456789|p0123456789|q0123456789|q0123456789|" "r0123456789|r0123456789|s0123456789|s0123456789|t0123456789|" "t0123456789|u0123456789|u0123456789|v0123456789|v0123456789|" "w0123456789|w0123456789|x0123456789|x0123456789|y0123456789|" "y0123456789|z0123456789|z0123456789)(a0123456789|a0123456789|" "b0123456789|b0123456789|c0123456789|c0123456789|d0123456789|" "d0123456789|e0123456789|e0123456789|f0123456789|f0123456789|" "g0123456789|g0123456789|h0123456789|h0123456789|i0123456789|" "i0123456789|j0123456789|j0123456789|k0123456789|k0123456789|" "l0123456789|l0123456789|m0123456789|m0123456789|n0123456789|" "n0123456789|o0123456789|o0123456789|p0123456789|p0123456789|" "q0123456789|q0123456789|r0123456789|r0123456789|s0123456789|" "s0123456789|t0123456789|t0123456789|u0123456789|u0123456789|" "v0123456789|v0123456789|w0123456789|w0123456789|x0123456789|" "x0123456789|y0123456789|y0123456789|z0123456789|z0123456789)" ) .Unwrap(); std::cout << "Initial Node Numbers:" << fsm_wse.GetFsm().NumStates() << std::endl; auto time_start = std::chrono::high_resolution_clock::now(); fsm_wse = fsm_wse.SimplifyEpsilon(); auto time_end = std::chrono::high_resolution_clock::now(); auto duration = std::chrono::duration_cast(time_end - time_start); std::cout << "Time taken to simplify epsilon: " << duration.count() << " ms" << std::endl; std::cout << "After SimplifyEpsilon Node Numbers:" << fsm_wse.GetFsm().NumStates() << std::endl; time_start = std::chrono::high_resolution_clock::now(); fsm_wse = fsm_wse.MergeEquivalentStates(); time_end = std::chrono::high_resolution_clock::now(); duration = std::chrono::duration_cast(time_end - time_start); std::cout << "Time taken to simplify transition: " << duration.count() << " ms" << std::endl; std::cout << "After SimplifyTransition Node Numbers:" << fsm_wse.GetFsm().NumStates() << std::endl; time_start = std::chrono::high_resolution_clock::now(); fsm_wse = std::move(fsm_wse.ToDFA()).Unwrap(); time_end = std::chrono::high_resolution_clock::now(); duration = std::chrono::duration_cast(time_end - time_start); std::cout << "Time taken to convert to DFA: " << duration.count() << " ms" << std::endl; std::cout << "After ToDFA Node Numbers:" << fsm_wse.GetFsm().NumStates() << std::endl; time_start = std::chrono::high_resolution_clock::now(); fsm_wse = std::move(fsm_wse.MinimizeDFA()).Unwrap(); time_end = std::chrono::high_resolution_clock::now(); duration = std::chrono::duration_cast(time_end - time_start); std::cout << "Time taken to minimize DFA: " << duration.count() << " ms" << std::endl; EXPECT_EQ(fsm_wse.GetFsm().NumStates(), 111); std::cout << "--------- Efficiency Test Passed! -----------" << std::endl; } TEST(XGrammarFSMTest, TestEmail) { std::string email_pattern = R"((\w+)(\.\w+)*@(\w+)(\.\w+)+)"; auto fsm_wse = RegexFSMBuilder::Build(email_pattern).Unwrap(); std::string valid_emails[5] = { "asnjdaj_19032910@google.com.test", "12393089340190@a.b.c.d.f.e.org.test", "as____________as@abc.me.test", "ooooohhhhh@123456.test", "ajidoa@a.test" }; for (const auto& email : valid_emails) { EXPECT_TRUE(fsm_wse.AcceptString(email)) << "Failed for email: " << email; } std::string invalid_emails[5] = { "@google.test", "hello@", "hello@.test", "+++asd@b.test", "hello" }; for (const auto& email : invalid_emails) { EXPECT_FALSE(fsm_wse.AcceptString(email)) << "Failed for email: " << email; } } TEST(XGrammarFSMTest, TestTime) { std::string time_pattern = R"((\d{1,2}):(\d{2})(:(\d{2}))?)"; auto fsm_wse = RegexFSMBuilder::Build(time_pattern).Unwrap(); std::string valid_times[5] = {"1:34", "23:59", "00:00", "01:02:03", "23:59:59"}; for (const auto& time : valid_times) { EXPECT_TRUE(fsm_wse.AcceptString(time)) << "Failed for time: " << time; } std::string invalid_times[9] = { "19", "12:6", "12:34:", "12:34:5", "12:34:567", "12:123", "12:", ":34:23", "::" }; for (const auto& time : invalid_times) { EXPECT_FALSE(fsm_wse.AcceptString(time)) << "Failed for time: " << time; } } TEST(XGrammarFSMTest, MergingNodesTest) { FSMWithStartEnd fsm_wse; for (int i = 0; i < 10; i++) { fsm_wse.AddState(); } fsm_wse.SetStartState(0); fsm_wse.AddEndState(9); fsm_wse.GetFsm().AddEdge(0, 1, 'a', 'a'); fsm_wse.GetFsm().AddEdge(0, 2, 'a', 'a'); fsm_wse.GetFsm().AddEdge(1, 3, 'b', 'b'); fsm_wse.GetFsm().AddEdge(1, 3, 'c', 'c'); fsm_wse.GetFsm().AddEdge(1, 4, 'b', 'b'); fsm_wse.GetFsm().AddEdge(1, 4, 'c', 'c'); fsm_wse.GetFsm().AddEdge(2, 5, 'b', 'b'); fsm_wse.GetFsm().AddEdge(2, 5, 'c', 'c'); fsm_wse.GetFsm().AddEdge(2, 6, 'b', 'b'); fsm_wse.GetFsm().AddEdge(2, 6, 'c', 'c'); fsm_wse.GetFsm().AddEdge(3, 7, 'd', 'd'); fsm_wse.GetFsm().AddEdge(4, 7, 'd', 'd'); fsm_wse.GetFsm().AddEdge(5, 8, 'd', 'd'); fsm_wse.GetFsm().AddEdge(6, 8, 'd', 'd'); fsm_wse.GetFsm().AddEdge(7, 9, 'e', 'e'); fsm_wse.GetFsm().AddEdge(8, 9, 'e', 'e'); fsm_wse = fsm_wse.MergeEquivalentStates(); std::string expected_fsm = R"(FSM(num_states=5, start=3, end=[4], edges=[ 0: ['d'->2] 1: ['b'->0, 'c'->0] 2: ['e'->4] 3: ['a'->1] 4: [] ]))"; EXPECT_EQ(fsm_wse.ToString(), expected_fsm); EXPECT_EQ(fsm_wse.GetFsm().NumStates(), 5); } TEST(XGrammarFSMTest, MergeEquivalentStatesNoCrossRuleChaining) { FSMWithStartEnd fsm_wse; for (int i = 0; i < 7; ++i) { fsm_wse.AddState(); } fsm_wse.SetStartState(0); fsm_wse.AddEndState(6); // 2 and 3 are equivalent successors of 0 under 'x' (Case 1). fsm_wse.GetFsm().AddEdge(0, 2, 'x', 'x'); fsm_wse.GetFsm().AddEdge(0, 3, 'x', 'x'); // 1 is another predecessor of 4 under 'a' (Case 2 candidate with 2). fsm_wse.GetFsm().AddEdge(0, 1, 'y', 'y'); fsm_wse.GetFsm().AddEdge(1, 4, 'a', 'a'); fsm_wse.GetFsm().AddEdge(2, 4, 'a', 'a'); fsm_wse.GetFsm().AddEdge(3, 5, 'b', 'b'); fsm_wse.GetFsm().AddEdge(4, 6, 'm', 'm'); fsm_wse.GetFsm().AddEdge(5, 6, 'n', 'n'); auto merged = fsm_wse.MergeEquivalentStates(); // Still accepts original strings. EXPECT_TRUE(merged.AcceptString("xam")); EXPECT_TRUE(merged.AcceptString("xbn")); EXPECT_TRUE(merged.AcceptString("yam")); // Should not over-merge and introduce this path. EXPECT_FALSE(merged.AcceptString("ybn")); } TEST(XGrammarFSMTest, EpsilonSimplificationTest) { FSMWithStartEnd fsm_wse; for (int i = 0; i < 10; i++) { fsm_wse.AddState(); } fsm_wse.SetStartState(0); fsm_wse.AddEndState(9); fsm_wse.GetFsm().AddEpsilonEdge(0, 1); fsm_wse.GetFsm().AddEpsilonEdge(0, 2); fsm_wse.GetFsm().AddEdge(1, 3, 'b', 'b'); fsm_wse.GetFsm().AddEpsilonEdge(1, 3); fsm_wse.GetFsm().AddEdge(1, 4, 'b', 'b'); fsm_wse.GetFsm().AddEdge(3, 3, 'c', 'c'); fsm_wse.GetFsm().AddEpsilonEdge(2, 5); fsm_wse.GetFsm().AddEdge(2, 5, 'c', 'c'); fsm_wse.GetFsm().AddEdge(2, 6, 'b', 'b'); fsm_wse.GetFsm().AddEdge(2, 6, 'c', 'c'); fsm_wse.GetFsm().AddEpsilonEdge(3, 7); fsm_wse.GetFsm().AddEpsilonEdge(4, 7); fsm_wse.GetFsm().AddEpsilonEdge(5, 8); fsm_wse.GetFsm().AddEpsilonEdge(6, 8); fsm_wse.GetFsm().AddEpsilonEdge(7, 9); fsm_wse.GetFsm().AddEpsilonEdge(8, 9); fsm_wse = fsm_wse.SimplifyEpsilon(); std::string expected_fsm = R"(FSM(num_states=3, start=0, end=[1], edges=[ 0: [Eps->1, Eps->2, 'b'->1, 'b'->2, 'c'->1] 1: [] 2: [Eps->1, 'c'->2] ]))"; EXPECT_EQ(fsm_wse.ToString(), expected_fsm); EXPECT_EQ(fsm_wse.GetFsm().NumStates(), 3); } xgrammar-0.2.3/tests/cpp/test_fsm_builder.cc000066400000000000000000000301431521764210300211210ustar00rootroot00000000000000/** * \file tests/cpp/test_fsm_builder.cc * \brief Test FSM builders: regex, trie, etc. */ #include #include #include #include "fsm.h" #include "fsm_builder.h" #include "grammar_functor.h" #include "xgrammar/grammar.h" using namespace xgrammar; TEST(XGrammarFSMBuilderTest, TestTrieFSMBuilder) { TrieFSMBuilder trie_builder; std::vector patterns = {"hello", "hi", "哈哈", "哈", "hili", "good"}; auto fsm_result = trie_builder.Build(patterns, {}); EXPECT_TRUE(fsm_result.has_value()); auto fsm = std::move(fsm_result).value(); // Test1: The printed result of FSM // Test2: The printed result of CompactFSM CompactFSMWithStartEnd compact_fsm(fsm.GetFsm().ToCompact(), fsm.GetStart(), fsm.GetEnds()); // Test3: Walk through the FSM int state = fsm.GetStart(); EXPECT_EQ(state, 0); // Test "hello" state = fsm.GetStart(); EXPECT_EQ(fsm.GetFsm().GetNextState(state, 'h'), 1); EXPECT_EQ(fsm.GetFsm().GetNextState(1, 'e'), 2); EXPECT_EQ(fsm.GetFsm().GetNextState(2, 'l'), 3); EXPECT_EQ(fsm.GetFsm().GetNextState(3, 'l'), 4); EXPECT_EQ(fsm.GetFsm().GetNextState(4, 'o'), 5); EXPECT_TRUE(fsm.IsEndState(5)); // Test "hil" state = fsm.GetStart(); EXPECT_EQ(fsm.GetFsm().GetNextState(state, 'h'), 1); EXPECT_EQ(fsm.GetFsm().GetNextState(1, 'i'), 6); EXPECT_EQ(fsm.GetFsm().GetNextState(6, 'l'), 13); EXPECT_FALSE(fsm.IsEndState(13)); // Test walk failure state = fsm.GetStart(); EXPECT_EQ(fsm.GetFsm().GetNextState(state, 'g'), 15); EXPECT_EQ(fsm.GetFsm().GetNextState(15, 'o'), 16); EXPECT_EQ(fsm.GetFsm().GetNextState(16, 'e'), -1); } TEST(XGrammarFSMBuilderTest, TestTagDispatchFSMBuilder1) { // Case 1. loop_after_dispatch = true Grammar::Impl::TagDispatch tag_dispatch = { /* tag_rule_pairs = */ {{"hel", 1}, {"hi", 2}, {"哈", 3}}, /* loop_after_dispatch = */ true, /* excludes = */ {} }; auto fsm_result = GrammarFSMBuilder::TagDispatch(tag_dispatch); EXPECT_TRUE(fsm_result.has_value()); auto fsm = std::move(fsm_result).value(); auto fsm_printed = fsm.ToString(); std::string expected_fsm_printed = R"(FSM(num_states=8, start=0, end=[0, 1, 2, 5, 6], edges=[ 0: [[\0-g]->0, 'h'->1, [i-\xe4]->0, '\xe5'->5, [\xe6-\xff]->0] 1: [[\0-d]->0, 'e'->2, [f-g]->0, 'h'->1, 'i'->4, [j-\xe4]->0, '\xe5'->5, [\xe6-\xff]->0] 2: [[\0-g]->0, 'h'->1, [i-k]->0, 'l'->3, [m-\xe4]->0, '\xe5'->5, [\xe6-\xff]->0] 3: [Rule(1)->0] 4: [Rule(2)->0] 5: [[\0-g]->0, 'h'->1, [i-\x92]->0, '\x93'->6, [\x94-\xe4]->0, '\xe5'->5, [\xe6-\xff]->0] 6: [[\0-g]->0, 'h'->1, [i-\x87]->0, '\x88'->7, [\x89-\xe4]->0, '\xe5'->5, [\xe6-\xff]->0] 7: [Rule(3)->0] ]))"; EXPECT_EQ(fsm_printed, expected_fsm_printed); } TEST(XGrammarFSMBuilderTest, TestTagDispatchFSMBuilder2) { // Case 2. loop_after_dispatch = false Grammar::Impl::TagDispatch tag_dispatch = { /* tag_rule_pairs = */ {{"hel", 1}, {"hi", 2}, {"哈", 3}}, /* loop_after_dispatch = */ false, /* excludes = */ {} }; auto fsm_result = GrammarFSMBuilder::TagDispatch(tag_dispatch); EXPECT_TRUE(fsm_result.has_value()); auto fsm = std::move(fsm_result).value(); auto fsm_printed = fsm.ToString(); std::string expected_fsm_printed = R"(FSM(num_states=11, start=0, end=[0, 1, 2, 5, 6, 8, 9, 10], edges=[ 0: [[\0-g]->0, 'h'->1, [i-\xe4]->0, '\xe5'->5, [\xe6-\xff]->0] 1: [[\0-d]->0, 'e'->2, [f-g]->0, 'h'->1, 'i'->4, [j-\xe4]->0, '\xe5'->5, [\xe6-\xff]->0] 2: [[\0-g]->0, 'h'->1, [i-k]->0, 'l'->3, [m-\xe4]->0, '\xe5'->5, [\xe6-\xff]->0] 3: [Rule(1)->8] 4: [Rule(2)->9] 5: [[\0-g]->0, 'h'->1, [i-\x92]->0, '\x93'->6, [\x94-\xe4]->0, '\xe5'->5, [\xe6-\xff]->0] 6: [[\0-g]->0, 'h'->1, [i-\x87]->0, '\x88'->7, [\x89-\xe4]->0, '\xe5'->5, [\xe6-\xff]->0] 7: [Rule(3)->10] 8: [] 9: [] 10: [] ]))"; EXPECT_EQ(fsm_printed, expected_fsm_printed); } TEST(XGrammarFSMBuilderTest, TestTagDispatchFSMBuilder3) { // Case 3. string excludes are compiled into the trie Grammar::Impl::TagDispatch tag_dispatch = { /* tag_rule_pairs = */ {{"hel", 1}, {"hi", 2}, {"哈", 3}}, /* loop_after_dispatch = */ true, /* excludes = */ {"hos", "eos"} }; auto fsm_result = GrammarFSMBuilder::TagDispatch(tag_dispatch); EXPECT_TRUE(fsm_result.has_value()); auto fsm = std::move(fsm_result).value(); auto fsm_printed = fsm.ToString(); EXPECT_NE(fsm_printed.find("Rule(1)->0"), std::string::npos); EXPECT_NE(fsm_printed.find("Rule(2)->0"), std::string::npos); EXPECT_NE(fsm_printed.find("Rule(3)->0"), std::string::npos); } TEST(XGrammarFSMBuilderTest, TestTokenTagDispatchFSMBuilder) { Grammar::Impl::TokenTagDispatch ttd = { /* trigger_rule_pairs = */ {{3, 1}, {5, 2}}, /* loop_after_dispatch = */ false, /* excludes = */ {7} }; auto fsm_result = GrammarFSMBuilder::TokenTagDispatch(ttd); EXPECT_TRUE(fsm_result.has_value()); auto fsm = std::move(fsm_result).value(); auto fsm_printed = fsm.ToString(); EXPECT_NE(fsm_printed.find("Token"), std::string::npos); EXPECT_NE(fsm_printed.find("ExcludeToken"), std::string::npos); } using GrammarExpr = Grammar::Impl::GrammarExpr; using GrammarExprType = Grammar::Impl::GrammarExprType; TEST(XGrammarFSMBuilderTest, TestByteStringFSMBuilder1) { int32_t byte_string[] = {'h', 'e', 'l', 'l', 'o'}; GrammarExpr grammar_expr = {GrammarExprType::kByteString, byte_string, 5}; auto fsm = GrammarFSMBuilder::ByteString(grammar_expr); auto fsm_printed = fsm.ToString(); std::string expected_fsm_printed = R"(FSM(num_states=6, start=0, end=[5], edges=[ 0: ['h'->1] 1: ['e'->2] 2: ['l'->3] 3: ['l'->4] 4: ['o'->5] 5: [] ]))"; EXPECT_EQ(fsm_printed, expected_fsm_printed); } TEST(XGrammarFSMBuilderTest, TestByteStringFSMBuilder2) { std::string byte_string = "你好"; std::vector byte_string_vec(byte_string.begin(), byte_string.end()); GrammarExpr grammar_expr = { GrammarExprType::kByteString, byte_string_vec.data(), static_cast(byte_string_vec.size()) }; auto fsm = GrammarFSMBuilder::ByteString(grammar_expr); auto fsm_printed = fsm.ToString(); std::string expected_fsm_printed = R"(FSM(num_states=7, start=0, end=[6], edges=[ 0: ['\xe4'->1] 1: ['\xbd'->2] 2: ['\xa0'->3] 3: ['\xe5'->4] 4: ['\xa5'->5] 5: ['\xbd'->6] 6: [] ]))"; EXPECT_EQ(fsm_printed, expected_fsm_printed); } TEST(XGrammarFSMBuilderTest, TestRuleRefFSMBuilder) { int32_t rule_ref = 1; GrammarExpr grammar_expr = {GrammarExprType::kRuleRef, &rule_ref, 1}; auto fsm = GrammarFSMBuilder::RuleRef(grammar_expr); auto fsm_printed = fsm.ToString(); std::string expected_fsm_printed = R"(FSM(num_states=2, start=0, end=[1], edges=[ 0: [Rule(1)->1] 1: [] ]))"; EXPECT_EQ(fsm_printed, expected_fsm_printed); } TEST(XGrammarFSMBuilderTest, TestCharacterClassFSMBuilder1) { std::vector datas = {0, 'a', 'z', 'A', 'Z'}; GrammarExpr grammar_expr = { GrammarExprType::kCharacterClass, datas.data(), static_cast(datas.size()) }; auto fsm = GrammarFSMBuilder::CharacterClass(grammar_expr); auto fsm_printed = fsm.ToString(); std::string expected_fsm_printed = R"(FSM(num_states=2, start=0, end=[1], edges=[ 0: [[a-z]->1, [A-Z]->1] 1: [] ]))"; EXPECT_EQ(fsm_printed, expected_fsm_printed); } TEST(XGrammarFSMBuilderTest, TestCharacterClassFSMBuilder2) { std::vector datas = {0, 'a', 'z', 'A', 'Z'}; GrammarExpr grammar_expr = { GrammarExprType::kCharacterClassStar, datas.data(), static_cast(datas.size()) }; auto fsm = GrammarFSMBuilder::CharacterClass(grammar_expr); auto fsm_printed = fsm.ToString(); std::string expected_fsm_printed = R"(FSM(num_states=1, start=0, end=[0], edges=[ 0: [[a-z]->0, [A-Z]->0] ]))"; EXPECT_EQ(fsm_printed, expected_fsm_printed); } TEST(XGrammarFSMBuilderTest, TestCharacterClassFSMBuilder3) { std::vector datas = {1, 'a', 'z', 'A', 'Z'}; GrammarExpr grammar_expr = { GrammarExprType::kCharacterClass, datas.data(), static_cast(datas.size()) }; auto fsm = GrammarFSMBuilder::CharacterClass(grammar_expr); auto fsm_printed = fsm.ToString(); std::string expected_fsm_printed = R"(FSM(num_states=8, start=0, end=[1], edges=[ 0: [[\0-@]->1, [[-`]->1, [{-\x7f]->1, [\xc0-\xdf]->2, [\xe0-\xef]->3, [\xf0-\xf7]->5] 1: [] 2: [[\x80-\xbf]->1] 3: [[\x80-\xbf]->4] 4: [[\x80-\xbf]->1] 5: [[\x80-\xbf]->6] 6: [[\x80-\xbf]->7] 7: [[\x80-\xbf]->1] ]))"; EXPECT_EQ(fsm_printed, expected_fsm_printed); } TEST(XGrammarFSMBuilderTest, TestCharacterClassFSMBuilder4) { std::vector datas = {1, 'a', 'z', 'A', 'Z'}; GrammarExpr grammar_expr = { GrammarExprType::kCharacterClassStar, datas.data(), static_cast(datas.size()) }; auto fsm = GrammarFSMBuilder::CharacterClass(grammar_expr); auto fsm_printed = fsm.ToString(); std::string expected_fsm_printed = R"(FSM(num_states=7, start=0, end=[0], edges=[ 0: [[\0-@]->0, [[-`]->0, [{-\x7f]->0, [\xc0-\xdf]->1, [\xe0-\xef]->2, [\xf0-\xf7]->4] 1: [[\x80-\xbf]->0] 2: [[\x80-\xbf]->3] 3: [[\x80-\xbf]->0] 4: [[\x80-\xbf]->5] 5: [[\x80-\xbf]->6] 6: [[\x80-\xbf]->0] ]))"; EXPECT_EQ(fsm_printed, expected_fsm_printed); } TEST(XGrammarFSMBuilderTest, TestSequenceFSMBuilder) { std::string test_grammar = R"( root ::= rule1 rule2 rule3 rule1 ::= "a" [a-z]* rule3 rule2 ::= "c" [A-Z] rule3 rule3 ::= "a" rule3 )"; auto grammar = Grammar::FromEBNF(test_grammar); std::string expected_fsm_root = R"(FSM(num_states=4, start=2, end=[3], edges=[ 0: [Rule(2)->1] 1: [Rule(3)->3] 2: [Rule(1)->0] 3: [] ]))"; auto fsm_root_result = GrammarFSMBuilder::Choices( grammar->GetGrammarExpr(grammar->GetRootRule().body_expr_id), grammar ); EXPECT_TRUE(fsm_root_result.has_value()); EXPECT_EQ(fsm_root_result->ToString(), expected_fsm_root); auto fsm_rule1_result = GrammarFSMBuilder::Choices( grammar->GetGrammarExpr(grammar->GetRule(1).body_expr_id), grammar ); std::string expected_fsm_rule1 = R"(FSM(num_states=3, start=1, end=[2], edges=[ 0: [Rule(3)->2, [a-z]->0] 1: ['a'->0] 2: [] ]))"; EXPECT_TRUE(fsm_rule1_result.has_value()); EXPECT_EQ(fsm_rule1_result->ToString(), expected_fsm_rule1); auto fsm_rule2_result = GrammarFSMBuilder::Choices( grammar->GetGrammarExpr(grammar->GetRule(2).body_expr_id), grammar ); std::string expected_fsm_rule2 = R"(FSM(num_states=4, start=2, end=[3], edges=[ 0: [[A-Z]->1] 1: [Rule(3)->3] 2: ['c'->0] 3: [] ]))"; EXPECT_TRUE(fsm_rule2_result.has_value()); EXPECT_EQ(fsm_rule2_result->ToString(), expected_fsm_rule2); auto fsm_rule3_result = GrammarFSMBuilder::Choices( grammar->GetGrammarExpr(grammar->GetRule(3).body_expr_id), grammar ); std::string expected_fsm_rule3 = R"(FSM(num_states=3, start=1, end=[2], edges=[ 0: [Rule(3)->2] 1: ['a'->0] 2: [] ]))"; EXPECT_TRUE(fsm_rule3_result.has_value()); EXPECT_EQ(fsm_rule3_result->ToString(), expected_fsm_rule3); } TEST(XGrammarFSMBuilderTest, TestChoicesFSMBuilder) { std::string test_grammar = R"( root ::= rule1 | rule2 rule1 ::= "" | "hello" rule2 rule2 ::= [a-z]* "A" | "B" rule2 )"; auto grammar = Grammar::FromEBNF(test_grammar); auto fsm_root_result = GrammarFSMBuilder::Choices( grammar->GetGrammarExpr(grammar->GetRootRule().body_expr_id), grammar ); std::string expected_fsm_root = R"(FSM(num_states=3, start=0, end=[1, 2], edges=[ 0: [Rule(1)->1, Rule(2)->2] 1: [] 2: [] ]))"; EXPECT_TRUE(fsm_root_result.has_value()); EXPECT_EQ(fsm_root_result->ToString(), expected_fsm_root); auto fsm_rule1_result = GrammarFSMBuilder::Choices( grammar->GetGrammarExpr(grammar->GetRule(1).body_expr_id), grammar ); std::string expected_fsm_rule1 = R"(FSM(num_states=7, start=0, end=[0, 6], edges=[ 0: ['h'->2] 1: [Rule(2)->6] 2: ['e'->3] 3: ['l'->4] 4: ['l'->5] 5: ['o'->1] 6: [] ]))"; EXPECT_TRUE(fsm_rule1_result.has_value()); EXPECT_EQ(fsm_rule1_result->ToString(), expected_fsm_rule1); auto fsm_rule2_result = GrammarFSMBuilder::Choices( grammar->GetGrammarExpr(grammar->GetRule(2).body_expr_id), grammar ); std::string expected_fsm_rule2 = R"(FSM(num_states=4, start=1, end=[0], edges=[ 0: [] 1: [Eps->2, 'B'->3] 2: ['A'->0, [a-z]->2] 3: [Rule(2)->0] ]))"; EXPECT_TRUE(fsm_rule2_result.has_value()); EXPECT_EQ(fsm_rule2_result->ToString(), expected_fsm_rule2); } xgrammar-0.2.3/tests/cpp/test_parser.cc000066400000000000000000000434421521764210300201300ustar00rootroot00000000000000#include #include #include "grammar_parser.h" #include "support/encoding.h" #include "test_utils.h" using namespace xgrammar; // Note: the inputs to the lexer tests may not be valid EBNF TEST(XGrammarLexerTest, BasicTokenization) { // Test basic token types std::string input = "rule1 ::= \"string\" | [a-z] | 123 | (expr) | {1,3} | * | + | ? | true | false"; EBNFLexer lexer; auto tokens = lexer.Tokenize(input); ASSERT_EQ(tokens.size(), 32); // 27 tokens + EOF // Check token types EXPECT_EQ(tokens[0].type, EBNFLexer::TokenType::RuleName); EXPECT_EQ(tokens[0].lexeme, "rule1"); EXPECT_EQ(tokens[1].type, EBNFLexer::TokenType::Assign); EXPECT_EQ(tokens[1].lexeme, "::="); EXPECT_EQ(tokens[2].type, EBNFLexer::TokenType::StringLiteral); EXPECT_EQ(tokens[2].lexeme, "\"string\""); XGRAMMAR_EXPECT_ANY_EQ(tokens[2].value, std::string, "string"); EXPECT_EQ(tokens[3].type, EBNFLexer::TokenType::Pipe); EXPECT_EQ(tokens[4].type, EBNFLexer::TokenType::LBracket); EXPECT_EQ(tokens[5].type, EBNFLexer::TokenType::CharInCharClass); EXPECT_EQ(tokens[5].lexeme, "a"); XGRAMMAR_EXPECT_ANY_EQ(tokens[5].value, TCodepoint, 'a'); EXPECT_EQ(tokens[6].type, EBNFLexer::TokenType::Dash); EXPECT_EQ(tokens[7].type, EBNFLexer::TokenType::CharInCharClass); EXPECT_EQ(tokens[7].lexeme, "z"); XGRAMMAR_EXPECT_ANY_EQ(tokens[7].value, TCodepoint, 'z'); EXPECT_EQ(tokens[8].type, EBNFLexer::TokenType::RBracket); EXPECT_EQ(tokens[9].type, EBNFLexer::TokenType::Pipe); EXPECT_EQ(tokens[10].type, EBNFLexer::TokenType::IntegerLiteral); EXPECT_EQ(tokens[10].lexeme, "123"); XGRAMMAR_EXPECT_ANY_EQ(tokens[10].value, int64_t, 123); EXPECT_EQ(tokens[11].type, EBNFLexer::TokenType::Pipe); EXPECT_EQ(tokens[12].type, EBNFLexer::TokenType::LParen); EXPECT_EQ(tokens[13].type, EBNFLexer::TokenType::Identifier); EXPECT_EQ(tokens[14].type, EBNFLexer::TokenType::RParen); EXPECT_EQ(tokens[15].type, EBNFLexer::TokenType::Pipe); EXPECT_EQ(tokens[16].type, EBNFLexer::TokenType::LBrace); EXPECT_EQ(tokens[17].type, EBNFLexer::TokenType::IntegerLiteral); EXPECT_EQ(tokens[18].type, EBNFLexer::TokenType::Comma); EXPECT_EQ(tokens[19].type, EBNFLexer::TokenType::IntegerLiteral); EXPECT_EQ(tokens[20].type, EBNFLexer::TokenType::RBrace); EXPECT_EQ(tokens[21].type, EBNFLexer::TokenType::Pipe); EXPECT_EQ(tokens[22].type, EBNFLexer::TokenType::Star); EXPECT_EQ(tokens[23].type, EBNFLexer::TokenType::Pipe); EXPECT_EQ(tokens[24].type, EBNFLexer::TokenType::Plus); EXPECT_EQ(tokens[25].type, EBNFLexer::TokenType::Pipe); EXPECT_EQ(tokens[26].type, EBNFLexer::TokenType::Question); EXPECT_EQ(tokens[27].type, EBNFLexer::TokenType::Pipe); EXPECT_EQ(tokens[28].type, EBNFLexer::TokenType::BooleanLiteral); EXPECT_EQ(tokens[28].lexeme, "true"); XGRAMMAR_EXPECT_ANY_EQ(tokens[28].value, bool, true); EXPECT_EQ(tokens[29].type, EBNFLexer::TokenType::Pipe); EXPECT_EQ(tokens[30].type, EBNFLexer::TokenType::BooleanLiteral); EXPECT_EQ(tokens[30].lexeme, "false"); XGRAMMAR_EXPECT_ANY_EQ(tokens[30].value, bool, false); EXPECT_EQ(tokens[31].type, EBNFLexer::TokenType::EndOfFile); } TEST(XGrammarLexerTest, CommentsAndWhitespace) { std::string input = "rule1 ::= expr1 # This is a comment\n | expr2 # Another comment"; EBNFLexer lexer; auto tokens = lexer.Tokenize(input); ASSERT_EQ(tokens.size(), 6); // 5 tokens + EOF EXPECT_EQ(tokens[0].type, EBNFLexer::TokenType::RuleName); EXPECT_EQ(tokens[0].lexeme, "rule1"); EXPECT_EQ(tokens[1].type, EBNFLexer::TokenType::Assign); EXPECT_EQ(tokens[2].type, EBNFLexer::TokenType::Identifier); EXPECT_EQ(tokens[2].lexeme, "expr1"); EXPECT_EQ(tokens[3].type, EBNFLexer::TokenType::Pipe); EXPECT_EQ(tokens[4].type, EBNFLexer::TokenType::Identifier); EXPECT_EQ(tokens[4].lexeme, "expr2"); } TEST(XGrammarLexerTest, StringLiterals) { // Test string literals with escape sequences std::string input = "rule ::= \"normal string\" | \"escaped \\\"quotes\\\"\" | \"\\n\\r\\t\\\\\""; EBNFLexer lexer; auto tokens = lexer.Tokenize(input); ASSERT_EQ(tokens.size(), 8); // 7 tokens + EOF EXPECT_EQ(tokens[2].type, EBNFLexer::TokenType::StringLiteral); XGRAMMAR_EXPECT_ANY_EQ(tokens[2].value, std::string, "normal string"); EXPECT_EQ(tokens[4].type, EBNFLexer::TokenType::StringLiteral); XGRAMMAR_EXPECT_ANY_EQ(tokens[4].value, std::string, "escaped \"quotes\""); EXPECT_EQ(tokens[6].type, EBNFLexer::TokenType::StringLiteral); XGRAMMAR_EXPECT_ANY_EQ(tokens[6].value, std::string, "\n\r\t\\"); } TEST(XGrammarLexerTest, CharacterClasses) { std::string input = "rule ::= [a-z] | [0-9] | [^a-z] | [\\-\\]\\\\] | [\\u0041-\\u005A] | [测试] | [\\t\\r\\n] | " "[\\b\\f]"; EBNFLexer lexer; auto tokens = lexer.Tokenize(input); ASSERT_EQ(tokens.size(), 49); // 45 tokens + EOF // [a-z] EXPECT_EQ(tokens[2].type, EBNFLexer::TokenType::LBracket); EXPECT_EQ(tokens[3].type, EBNFLexer::TokenType::CharInCharClass); EXPECT_EQ(tokens[3].lexeme, "a"); XGRAMMAR_EXPECT_ANY_EQ(tokens[3].value, TCodepoint, 'a'); EXPECT_EQ(tokens[4].type, EBNFLexer::TokenType::Dash); EXPECT_EQ(tokens[5].type, EBNFLexer::TokenType::CharInCharClass); EXPECT_EQ(tokens[5].lexeme, "z"); XGRAMMAR_EXPECT_ANY_EQ(tokens[5].value, TCodepoint, 'z'); EXPECT_EQ(tokens[6].type, EBNFLexer::TokenType::RBracket); EXPECT_EQ(tokens[7].type, EBNFLexer::TokenType::Pipe); // [0-9] EXPECT_EQ(tokens[8].type, EBNFLexer::TokenType::LBracket); EXPECT_EQ(tokens[9].type, EBNFLexer::TokenType::CharInCharClass); EXPECT_EQ(tokens[9].lexeme, "0"); XGRAMMAR_EXPECT_ANY_EQ(tokens[9].value, TCodepoint, '0'); EXPECT_EQ(tokens[10].type, EBNFLexer::TokenType::Dash); EXPECT_EQ(tokens[11].type, EBNFLexer::TokenType::CharInCharClass); EXPECT_EQ(tokens[11].lexeme, "9"); XGRAMMAR_EXPECT_ANY_EQ(tokens[11].value, TCodepoint, '9'); EXPECT_EQ(tokens[12].type, EBNFLexer::TokenType::RBracket); EXPECT_EQ(tokens[13].type, EBNFLexer::TokenType::Pipe); // [^a-z] EXPECT_EQ(tokens[14].type, EBNFLexer::TokenType::LBracket); EXPECT_EQ(tokens[15].type, EBNFLexer::TokenType::Caret); EXPECT_EQ(tokens[16].type, EBNFLexer::TokenType::CharInCharClass); EXPECT_EQ(tokens[16].lexeme, "a"); XGRAMMAR_EXPECT_ANY_EQ(tokens[16].value, TCodepoint, 'a'); EXPECT_EQ(tokens[17].type, EBNFLexer::TokenType::Dash); EXPECT_EQ(tokens[18].type, EBNFLexer::TokenType::CharInCharClass); EXPECT_EQ(tokens[18].lexeme, "z"); XGRAMMAR_EXPECT_ANY_EQ(tokens[18].value, TCodepoint, 'z'); EXPECT_EQ(tokens[19].type, EBNFLexer::TokenType::RBracket); EXPECT_EQ(tokens[20].type, EBNFLexer::TokenType::Pipe); // [\-\]\\] EXPECT_EQ(tokens[21].type, EBNFLexer::TokenType::LBracket); EXPECT_EQ(tokens[22].type, EBNFLexer::TokenType::CharInCharClass); EXPECT_EQ(tokens[22].lexeme, "\\-"); XGRAMMAR_EXPECT_ANY_EQ(tokens[22].value, TCodepoint, '-'); EXPECT_EQ(tokens[23].type, EBNFLexer::TokenType::CharInCharClass); EXPECT_EQ(tokens[23].lexeme, "\\]"); XGRAMMAR_EXPECT_ANY_EQ(tokens[23].value, TCodepoint, ']'); EXPECT_EQ(tokens[24].type, EBNFLexer::TokenType::CharInCharClass); EXPECT_EQ(tokens[24].lexeme, "\\\\"); XGRAMMAR_EXPECT_ANY_EQ(tokens[24].value, TCodepoint, '\\'); EXPECT_EQ(tokens[25].type, EBNFLexer::TokenType::RBracket); EXPECT_EQ(tokens[26].type, EBNFLexer::TokenType::Pipe); // [\u0041-\u005A] EXPECT_EQ(tokens[27].type, EBNFLexer::TokenType::LBracket); EXPECT_EQ(tokens[28].type, EBNFLexer::TokenType::CharInCharClass); EXPECT_EQ(tokens[28].lexeme, "\\u0041"); XGRAMMAR_EXPECT_ANY_EQ(tokens[28].value, TCodepoint, 0x41); // 'A' EXPECT_EQ(tokens[29].type, EBNFLexer::TokenType::Dash); EXPECT_EQ(tokens[30].type, EBNFLexer::TokenType::CharInCharClass); EXPECT_EQ(tokens[30].lexeme, "\\u005A"); XGRAMMAR_EXPECT_ANY_EQ(tokens[30].value, TCodepoint, 0x5A); // 'Z' EXPECT_EQ(tokens[31].type, EBNFLexer::TokenType::RBracket); EXPECT_EQ(tokens[32].type, EBNFLexer::TokenType::Pipe); // [测试] EXPECT_EQ(tokens[33].type, EBNFLexer::TokenType::LBracket); EXPECT_EQ(tokens[34].type, EBNFLexer::TokenType::CharInCharClass); EXPECT_EQ(tokens[34].lexeme, "测"); XGRAMMAR_EXPECT_ANY_EQ(tokens[34].value, TCodepoint, 0x6D4B); EXPECT_EQ(tokens[35].type, EBNFLexer::TokenType::CharInCharClass); EXPECT_EQ(tokens[35].lexeme, "试"); XGRAMMAR_EXPECT_ANY_EQ(tokens[35].value, TCodepoint, 0x8BD5); EXPECT_EQ(tokens[36].type, EBNFLexer::TokenType::RBracket); EXPECT_EQ(tokens[37].type, EBNFLexer::TokenType::Pipe); // [\t\r\n] EXPECT_EQ(tokens[38].type, EBNFLexer::TokenType::LBracket); EXPECT_EQ(tokens[39].type, EBNFLexer::TokenType::CharInCharClass); EXPECT_EQ(tokens[39].lexeme, "\\t"); XGRAMMAR_EXPECT_ANY_EQ(tokens[39].value, TCodepoint, '\t'); EXPECT_EQ(tokens[40].type, EBNFLexer::TokenType::CharInCharClass); EXPECT_EQ(tokens[40].lexeme, "\\r"); XGRAMMAR_EXPECT_ANY_EQ(tokens[40].value, TCodepoint, '\r'); EXPECT_EQ(tokens[41].type, EBNFLexer::TokenType::CharInCharClass); EXPECT_EQ(tokens[41].lexeme, "\\n"); XGRAMMAR_EXPECT_ANY_EQ(tokens[41].value, TCodepoint, '\n'); EXPECT_EQ(tokens[42].type, EBNFLexer::TokenType::RBracket); EXPECT_EQ(tokens[43].type, EBNFLexer::TokenType::Pipe); // [\b\f] EXPECT_EQ(tokens[44].type, EBNFLexer::TokenType::LBracket); EXPECT_EQ(tokens[45].type, EBNFLexer::TokenType::CharInCharClass); EXPECT_EQ(tokens[45].lexeme, "\\b"); XGRAMMAR_EXPECT_ANY_EQ(tokens[45].value, TCodepoint, '\b'); EXPECT_EQ(tokens[46].type, EBNFLexer::TokenType::CharInCharClass); EXPECT_EQ(tokens[46].lexeme, "\\f"); XGRAMMAR_EXPECT_ANY_EQ(tokens[46].value, TCodepoint, '\f'); EXPECT_EQ(tokens[47].type, EBNFLexer::TokenType::RBracket); } TEST(XGrammarLexerTest, BooleanValues) { std::string input = "rule ::= true | false"; EBNFLexer lexer; auto tokens = lexer.Tokenize(input); ASSERT_EQ(tokens.size(), 6); // 5 tokens + EOF EXPECT_EQ(tokens[2].type, EBNFLexer::TokenType::BooleanLiteral); EXPECT_EQ(tokens[2].lexeme, "true"); XGRAMMAR_EXPECT_ANY_EQ(tokens[2].value, bool, true); EXPECT_EQ(tokens[4].type, EBNFLexer::TokenType::BooleanLiteral); EXPECT_EQ(tokens[4].lexeme, "false"); XGRAMMAR_EXPECT_ANY_EQ(tokens[4].value, bool, false); } TEST(XGrammarLexerTest, LookaheadAssertion) { std::string input = "rule ::= \"a\" (= lookahead)"; EBNFLexer lexer; auto tokens = lexer.Tokenize(input); ASSERT_EQ(tokens.size(), 7); // 6 tokens + EOF EXPECT_EQ(tokens[3].type, EBNFLexer::TokenType::LookaheadLParen); EXPECT_EQ(tokens[3].lexeme, "(="); EXPECT_EQ(tokens[5].type, EBNFLexer::TokenType::RParen); } TEST(XGrammarLexerTest, LineAndColumnTracking) { std::string input = "rule1 ::= expr1\nrule2 ::= expr2"; EBNFLexer lexer; auto tokens = lexer.Tokenize(input); ASSERT_EQ(tokens.size(), 7); // 6 tokens + EOF // First line tokens EXPECT_EQ(tokens[0].line, 1); EXPECT_EQ(tokens[0].column, 1); EXPECT_EQ(tokens[1].line, 1); EXPECT_EQ(tokens[1].column, 7); EXPECT_EQ(tokens[2].line, 1); EXPECT_EQ(tokens[2].column, 11); // Second line tokens EXPECT_EQ(tokens[3].line, 2); EXPECT_EQ(tokens[3].column, 1); EXPECT_EQ(tokens[4].line, 2); EXPECT_EQ(tokens[4].column, 7); EXPECT_EQ(tokens[5].line, 2); EXPECT_EQ(tokens[5].column, 11); } TEST(XGrammarLexerTest, ComplexGrammar) { std::string input = "# JSON Grammar\n" "root ::= value\n" "value ::= object | array | string | number | \"true\" | \"false\" | \"null\"\n" "object ::= \"{\" (member (\",\" member)*)? \"}\"\n" "member ::= string \":\" value\n" "array ::= \"[\" (value (\",\" value)*)? \"]\"\n" "string ::= \"\\\"\" char* \"\\\"\"\n" "char ::= [^\"\\\\] | \"\\\\\\\"\"\n" "number ::= int frac? exp?\n" "int ::= \"-\"? ([1-9] [0-9]* | \"0\")\n" "frac ::= \".\" [0-9]+\n" "exp ::= [eE] [+\\-]? [0-9]+"; EBNFLexer lexer; auto tokens = lexer.Tokenize(input); // Just verify we have a reasonable number of tokens and no crashes EXPECT_GT(tokens.size(), 50); EXPECT_EQ(tokens.back().type, EBNFLexer::TokenType::EndOfFile); } TEST(XGrammarLexerTest, EdgeCases) { // Empty input { std::string input = ""; EBNFLexer lexer; auto tokens = lexer.Tokenize(input); ASSERT_EQ(tokens.size(), 1); EXPECT_EQ(tokens[0].type, EBNFLexer::TokenType::EndOfFile); } // Only whitespace and comments { std::string input = " \t\n # Comment\n # Another comment"; EBNFLexer lexer; auto tokens = lexer.Tokenize(input); ASSERT_EQ(tokens.size(), 1); EXPECT_EQ(tokens[0].type, EBNFLexer::TokenType::EndOfFile); } // Various newline formats { std::string input = "rule1 ::= expr1\nrule2 ::= expr2\r\nrule3 ::= expr3\rrule4 ::= expr4"; EBNFLexer lexer; auto tokens = lexer.Tokenize(input); ASSERT_EQ(tokens.size(), 13); // 12 tokens + EOF } // Integer boundary { std::string input = "rule ::= 999999999999999"; // 15 digits (max allowed) EBNFLexer lexer; auto tokens = lexer.Tokenize(input); ASSERT_EQ(tokens.size(), 4); // 3 tokens + EOF EXPECT_EQ(tokens[2].type, EBNFLexer::TokenType::IntegerLiteral); EXPECT_EQ(tokens[2].lexeme, "999999999999999"); } // Special identifiers { std::string input = "rule-name ::= _special.identifier-123"; EBNFLexer lexer; auto tokens = lexer.Tokenize(input); ASSERT_EQ(tokens.size(), 4); // 3 tokens + EOF EXPECT_EQ(tokens[0].type, EBNFLexer::TokenType::RuleName); EXPECT_EQ(tokens[0].lexeme, "rule-name"); EXPECT_EQ(tokens[2].type, EBNFLexer::TokenType::Identifier); EXPECT_EQ(tokens[2].lexeme, "_special.identifier-123"); } } TEST(XGrammarLexerTest, QuantifierTokens) { std::string input = "rule ::= expr? | expr* | expr+ | expr{1} | expr{1,} | expr{1,5}"; EBNFLexer lexer; auto tokens = lexer.Tokenize(input); // Verify question mark, star, plus, and brace tokens EXPECT_EQ(tokens[3].type, EBNFLexer::TokenType::Question); EXPECT_EQ(tokens[6].type, EBNFLexer::TokenType::Star); EXPECT_EQ(tokens[9].type, EBNFLexer::TokenType::Plus); EXPECT_EQ(tokens[12].type, EBNFLexer::TokenType::LBrace); EXPECT_EQ(tokens[13].type, EBNFLexer::TokenType::IntegerLiteral); EXPECT_EQ(tokens[14].type, EBNFLexer::TokenType::RBrace); EXPECT_EQ(tokens[17].type, EBNFLexer::TokenType::LBrace); EXPECT_EQ(tokens[18].type, EBNFLexer::TokenType::IntegerLiteral); EXPECT_EQ(tokens[19].type, EBNFLexer::TokenType::Comma); EXPECT_EQ(tokens[20].type, EBNFLexer::TokenType::RBrace); } // Test for UTF-8 handling in string literals TEST(XGrammarLexerTest, UTF8Handling) { std::string input = "rule ::= \"UTF-8: \\u00A9 \\u2603 \\U0001F600\""; EBNFLexer lexer; auto tokens = lexer.Tokenize(input); ASSERT_EQ(tokens.size(), 4); // 3 tokens + EOF EXPECT_EQ(tokens[2].type, EBNFLexer::TokenType::StringLiteral); // The value should contain the actual UTF-8 characters XGRAMMAR_EXPECT_ANY_EQ(tokens[2].value, std::string, "UTF-8: © ☃ 😀"); } TEST(XGrammarLexerTest, LexerErrorCases) { // Test for unterminated string { std::string input = "rule ::= \"unterminated string"; XGRAMMAR_EXPECT_THROW( EBNFLexer().Tokenize(input), std::exception, "Expect \" in string literal" ); } // Test for unterminated character class { std::string input = "rule ::= [a-z"; XGRAMMAR_EXPECT_THROW( EBNFLexer().Tokenize(input), std::exception, "Unterminated character class" ); } // Test for unterminated character class with escaped bracket { std::string input = "rule ::= [a-z\\-\\\\\\]"; XGRAMMAR_EXPECT_THROW( EBNFLexer().Tokenize(input), std::exception, "Unterminated character class" ); } // Test for invalid UTF-8 sequence in string { std::string input = "rule ::= \"\xC2\x20\""; // Invalid UTF-8 sequence XGRAMMAR_EXPECT_THROW(EBNFLexer().Tokenize(input), std::exception, "Invalid UTF8 sequence"); } // Test for invalid escape sequence in string { std::string input = "rule ::= \"\\z\""; // Invalid escape sequence XGRAMMAR_EXPECT_THROW(EBNFLexer().Tokenize(input), std::exception, "Invalid escape sequence"); } // Test for newline in character class { std::string input = "rule ::= [a-z\n]"; XGRAMMAR_EXPECT_THROW( EBNFLexer().Tokenize(input), std::exception, "Character class should not contain newline" ); } // Test for invalid UTF-8 sequence in character class { std::string input = "rule ::= [\xC2\x20]"; // Invalid UTF-8 sequence XGRAMMAR_EXPECT_THROW(EBNFLexer().Tokenize(input), std::exception, "Invalid UTF8 sequence"); } // Test for invalid escape sequence in character class { std::string input = "rule ::= [\\z]"; // Invalid escape sequence XGRAMMAR_EXPECT_THROW(EBNFLexer().Tokenize(input), std::exception, "Invalid escape sequence"); } // Test for integer too large { std::string input = "rule ::= expr{1000000000000000000}"; // Integer > 1e15 XGRAMMAR_EXPECT_THROW(EBNFLexer().Tokenize(input), std::exception, "Integer is too large"); } // Test for unexpected character { std::string input = "rule ::= @"; XGRAMMAR_EXPECT_THROW(EBNFLexer().Tokenize(input), std::exception, "Unexpected character"); } // Test for unexpected colon { std::string input = "rule : expr"; XGRAMMAR_EXPECT_THROW(EBNFLexer().Tokenize(input), std::exception, "Unexpected character: ':'"); } // Test for assign preceded by non-identifier { std::string input = "\"string\" ::= expr"; XGRAMMAR_EXPECT_THROW( EBNFLexer().Tokenize(input), std::exception, "Assign should be preceded by an identifier" ); } // Test for assign as first token { std::string input = "::= expr"; XGRAMMAR_EXPECT_THROW( EBNFLexer().Tokenize(input), std::exception, "Assign should not be the first token" ); } // Test for rule name not at beginning of line { std::string input = "token token ::= expr"; XGRAMMAR_EXPECT_THROW( EBNFLexer().Tokenize(input), std::exception, "The rule name should be at the beginning of the line" ); } } xgrammar-0.2.3/tests/cpp/test_repetition_range.cc000066400000000000000000000020341521764210300221620ustar00rootroot00000000000000/*! * Copyright (c) 2026 by Contributors * \file tests/cpp/test_repetition_range.cc * \brief Regression test for RepetitionRangeExpander segfault when * grammar_expr_id (a builder ID) is looked up in base_grammar_. */ #include #include #include "grammar_functor.h" using namespace xgrammar; // HandleRepetitionRange looks up grammar_expr_id (a builder ID) in // base_grammar_ instead of builder_. The first expansion inflates the // builder's ID space; the second expansion's grammar_expr_id then // exceeds base_grammar_'s expression count, causing an OOB / segfault. TEST(RepetitionRangeExpanderTest, UnboundedRepetitionAboveThresholdDoesNotCrash) { // Two unbounded repeats with lower > kUnzipThreshold (128) in the // same rule. The first expand creates many builder expressions; the // second's grammar_expr_id lands out of bounds in base_grammar_. auto grammar = Grammar::FromEBNF(R"(root ::= [a-z]{129,} [0-9]{129,})"); EXPECT_NO_THROW(RepetitionRangeExpander::Apply(grammar)); } xgrammar-0.2.3/tests/cpp/test_serialization.cc000066400000000000000000000532431521764210300215110ustar00rootroot00000000000000#include #include #include #include #include #include #include #include "fsm.h" #include "support/compact_2d_array.h" #include "support/dynamic_bitset.h" #include "support/json_serializer.h" namespace xgrammar { bool operator==(const Compact2DArray& lhs, const Compact2DArray& rhs) { if (lhs.size() != rhs.size()) return false; const std::size_t indptr_size = lhs.size() + 1; const auto* lhs_indptr = lhs.indptr(); const auto* rhs_indptr = rhs.indptr(); for (std::size_t i = 0; i < indptr_size; ++i) { if (lhs_indptr[i] != rhs_indptr[i]) return false; } const auto data_size = *std::max_element(lhs_indptr, lhs_indptr + indptr_size); const auto* lhs_data = lhs.data(); const auto* rhs_data = rhs.data(); for (int i = 0; i < int(data_size); ++i) { if (lhs_data[i] != rhs_data[i]) return false; } return true; } bool operator==(const Compact2DArray& lhs, const Compact2DArray& rhs) { if (lhs.size() != rhs.size()) return false; const std::size_t indptr_size = lhs.size() + 1; const auto* lhs_indptr = lhs.indptr(); const auto* rhs_indptr = rhs.indptr(); for (std::size_t i = 0; i < indptr_size; ++i) { if (lhs_indptr[i] != rhs_indptr[i]) return false; } const auto data_size = *std::max_element(lhs_indptr, lhs_indptr + indptr_size); const auto* lhs_data = lhs.data(); const auto* rhs_data = rhs.data(); for (int i = 0; i < int(data_size); ++i) { if (!(lhs_data[i] == rhs_data[i])) return false; } return true; } } // namespace xgrammar TEST(XGrammarSerializationTest, TestSTLAndBuiltinTypes) { using namespace xgrammar; // Test basic types { bool value = true; auto json_value = AutoSerializeJSONValue(value); ASSERT_TRUE(json_value.is()); ASSERT_EQ(json_value.get(), true); // Test literal string comparison std::string expected = "true"; ASSERT_EQ(json_value.serialize(), expected); bool deserialized = false; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_EQ(deserialized, value); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } { int value = 42; auto json_value = AutoSerializeJSONValue(value); ASSERT_TRUE(json_value.is()); ASSERT_EQ(json_value.get(), 42); // Test literal string comparison std::string expected = "42"; ASSERT_EQ(json_value.serialize(), expected); int deserialized = 0; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_EQ(deserialized, value); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } { double value = 3.14; auto json_value = AutoSerializeJSONValue(value); ASSERT_TRUE(json_value.is()); ASSERT_EQ(json_value.get(), 3.14); // Test literal string comparison // due to precision, we can't compare strings directly // because it might serialize as "3.1400000000000001" or similar // so we compare the numeric value instead std::string expected = "3.14"; ASSERT_EQ(std::stod(json_value.serialize()), std::stod(expected)); double deserialized = 0.0; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_EQ(deserialized, value); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } { std::string value = "hello"; auto json_value = AutoSerializeJSONValue(value); ASSERT_TRUE(json_value.is()); ASSERT_EQ(json_value.get(), "hello"); // Test literal string comparison std::string expected = "\"hello\""; ASSERT_EQ(json_value.serialize(), expected); std::string deserialized; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_EQ(deserialized, value); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } // Test containers { std::vector value = {1, 2, 3}; auto json_value = AutoSerializeJSONValue(value); ASSERT_TRUE(json_value.is()); // Test literal string comparison std::string expected = "[1,2,3]"; ASSERT_EQ(json_value.serialize(), expected); std::vector deserialized; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_EQ(deserialized, value); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } { std::unordered_set value = {1, 2, 3}; auto json_value = AutoSerializeJSONValue(value); ASSERT_TRUE(json_value.is()); // Test literal string comparison (sorted due to unordered_set) std::string expected = "[1,2,3]"; ASSERT_EQ(json_value.serialize(), expected); std::unordered_set deserialized; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_EQ(deserialized, value); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } { std::pair value = {42, "hello"}; auto json_value = AutoSerializeJSONValue(value); ASSERT_TRUE(json_value.is()); // Test literal string comparison std::string expected = "[42,\"hello\"]"; ASSERT_EQ(json_value.serialize(), expected); std::pair deserialized; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_EQ(deserialized, value); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } // Test optional { std::optional value = 42; auto json_value = AutoSerializeJSONValue(value); ASSERT_TRUE(json_value.is()); ASSERT_EQ(json_value.get(), 42); // Test literal string comparison std::string expected = "42"; ASSERT_EQ(json_value.serialize(), expected); std::optional deserialized; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_TRUE(deserialized.has_value()); ASSERT_EQ(*deserialized, 42); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } { std::optional value; auto json_value = AutoSerializeJSONValue(value); ASSERT_TRUE(json_value.is()); // Test literal string comparison std::string expected = "null"; ASSERT_EQ(json_value.serialize(), expected); std::optional deserialized = 999; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_FALSE(deserialized.has_value()); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } } TEST(XGrammarSerializationTest, TestString) { using namespace xgrammar; { std::string value = "hello\nworld"; auto json_value = AutoSerializeJSONValue(value); ASSERT_EQ(json_value.serialize(), "\"hello\\nworld\""); std::string deserialized; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_EQ(deserialized, value); } { std::string value = "\xC3\x28"; auto json_value = AutoSerializeJSON(value); ASSERT_EQ(json_value, u8"\"\u00c3(\""); std::string deserialized; auto error = AutoDeserializeJSON(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_EQ(deserialized, value); } { std::string value = "我"; auto json_value = AutoSerializeJSON(value); std::cout << json_value << std::endl; ASSERT_EQ(json_value, u8"\"\u00e6\u0088\u0091\""); std::string deserialized; auto error = AutoDeserializeJSON(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_EQ(deserialized, value); } } TEST(XGrammarSerializationTest, TestFSMEdge) { using namespace xgrammar; // Test basic FSMEdge { FSMEdge edge{1, 2, 3}; auto json_value = AutoSerializeJSONValue(edge); ASSERT_TRUE(json_value.is()); // Test literal string comparison std::string expected = "[1,2,3]"; ASSERT_EQ(json_value.serialize(), expected); FSMEdge deserialized; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_EQ(deserialized, edge); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } // Test special edge types { FSMEdge epsilon_edge{FSMEdge::EdgeType::kEpsilon, 0, 5}; auto json_value = AutoSerializeJSONValue(epsilon_edge); // Test literal string comparison std::string expected = "[-1,0,5]"; ASSERT_EQ(json_value.serialize(), expected); FSMEdge deserialized; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_EQ(deserialized, epsilon_edge); ASSERT_TRUE(deserialized.IsEpsilon()); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } { FSMEdge rule_edge{FSMEdge::EdgeType::kRuleRef, 10, 7}; auto json_value = AutoSerializeJSONValue(rule_edge); // Test literal string comparison std::string expected = "[-2,10,7]"; ASSERT_EQ(json_value.serialize(), expected); FSMEdge deserialized; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_EQ(deserialized, rule_edge); ASSERT_TRUE(deserialized.IsRuleRef()); ASSERT_EQ(deserialized.GetRefRuleId(), 10); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } } TEST(XGrammarSerializationTest, TestCompact2DArray) { using namespace xgrammar; // Test empty array { Compact2DArray array; auto json_value = AutoSerializeJSONValue(array); ASSERT_TRUE(json_value.is()); // Test literal string comparison std::string expected = "{\"data_\":[],\"indptr_\":[0]}"; ASSERT_EQ(json_value.serialize(), expected); Compact2DArray deserialized; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_EQ(deserialized.size(), 0); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } // Test non-empty array { Compact2DArray array; array.PushBack({0, 1, 2, 3}); array.PushBack({4, 5, 6, 7}); array.PushBack({8, 9}); auto json_value = AutoSerializeJSONValue(array); ASSERT_TRUE(json_value.is()); // Test literal string comparison std::string expected = "{\"data_\":[0,1,2,3,4,5,6,7,8,9],\"indptr_\":[0,4,8,10]}"; ASSERT_EQ(json_value.serialize(), expected); // Check JSON structure const auto& obj = json_value.get(); ASSERT_TRUE(obj.find("data_") != obj.end()); ASSERT_TRUE(obj.find("indptr_") != obj.end()); const auto& data_array = obj.at("data_").get(); ASSERT_EQ(data_array.size(), 10); const auto& indptr_array = obj.at("indptr_").get(); ASSERT_EQ(indptr_array.size(), 4); // 3 rows + 1 Compact2DArray deserialized; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_EQ(deserialized, array); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } // Test with FSMEdge { Compact2DArray array; array.PushBack({{1, 2, 3}, {4, 5, 6}}); array.PushBack({{FSMEdge::EdgeType::kEpsilon, 0, 7}}); auto json_value = AutoSerializeJSONValue(array); ASSERT_TRUE(json_value.is()); // Test literal string comparison std::string expected = "{\"data_\":[[1,2,3],[4,5,6],[-1,0,7]],\"indptr_\":[0,2,3]}"; ASSERT_EQ(json_value.serialize(), expected); Compact2DArray deserialized; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_TRUE(deserialized == array); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } } TEST(XGrammarSerializationTest, TestDynamicBitset) { using namespace xgrammar; // Test empty bitset { DynamicBitset bitset(0); auto json_value = AutoSerializeJSONValue(bitset); ASSERT_TRUE(json_value.is()); // Test literal string comparison std::string expected = "[0,0]"; ASSERT_EQ(json_value.serialize(), expected); DynamicBitset deserialized; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_EQ(deserialized, bitset); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } // Test non-empty bitset { DynamicBitset bitset(64); bitset.Set(0); bitset.Set(10); bitset.Set(63); auto json_value = AutoSerializeJSONValue(bitset); ASSERT_TRUE(json_value.is()); // Test literal string comparison std::string expected = "[64,2,1025,2147483648]"; ASSERT_EQ(json_value.serialize(), expected); const auto& arr = json_value.get(); ASSERT_EQ(arr.size(), 4); // size, buffer_size, data[0], data[1] ASSERT_EQ(arr[0].get(), 64); // size ASSERT_EQ(arr[1].get(), 2); // buffer_size DynamicBitset deserialized; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_EQ(deserialized, bitset); // Verify bit values ASSERT_TRUE(deserialized[0]); ASSERT_TRUE(deserialized[10]); ASSERT_TRUE(deserialized[63]); ASSERT_FALSE(deserialized[1]); ASSERT_FALSE(deserialized[62]); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } // Test smaller bitset { DynamicBitset bitset(10); bitset.Set(0); bitset.Set(5); bitset.Set(9); auto json_value = AutoSerializeJSONValue(bitset); ASSERT_TRUE(json_value.is()); // Test literal string comparison // Bits 0, 5, 9 are set: 2^0 + 2^5 + 2^9 = 1 + 32 + 512 = 545 std::string expected = "[10,1,545]"; ASSERT_EQ(json_value.serialize(), expected); DynamicBitset deserialized; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_EQ(deserialized, bitset); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } } TEST(XGrammarSerializationTest, TestCompactFSM) { using namespace xgrammar; // Test simple FSM { FSM fsm(3); fsm.AddEdge(0, 1, 'a', 'a'); fsm.AddEdge(1, 2, 'b', 'b'); fsm.AddEpsilonEdge(0, 2); CompactFSM compact_fsm = fsm.ToCompact(); auto json_value = AutoSerializeJSONValue(compact_fsm); ASSERT_TRUE(json_value.is()); // Test literal string comparison - edges are sorted by CompactFSM std::string expected = "{\"edges\":{\"data_\":[[-1,0,2],[97,97,1],[98,98,2]],\"indptr_\":[0,2,3,3]},\"edge_aux_" "data\":[],\"edge_num\":3}"; ASSERT_EQ(json_value.serialize(), expected); CompactFSM deserialized; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); // Test basic properties ASSERT_EQ(deserialized.NumStates(), compact_fsm.NumStates()); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } // Test FSM with rule references { FSM fsm(3); fsm.AddEdge(0, 1, 'a', 'z'); fsm.AddRuleEdge(1, 2, 5); fsm.AddEOSEdge(2, 0); CompactFSM compact_fsm = fsm.ToCompact(); auto json_value = AutoSerializeJSONValue(compact_fsm); // Test literal string comparison std::string expected = "{\"edges\":{\"data_\":[[97,122,1],[-2,5,2],[-3,0,0]],\"indptr_\":[0,1,2,3]},\"edge_aux_" "data\":[],\"edge_num\":3}"; ASSERT_EQ(json_value.serialize(), expected); CompactFSM deserialized; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_EQ(deserialized.NumStates(), compact_fsm.NumStates()); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } } TEST(XGrammarSerializationTest, TestCompactFSMWithStartEnd) { using namespace xgrammar; // Test simple FSM { FSM fsm(3); fsm.AddEdge(0, 1, 'a', 'a'); fsm.AddEdge(1, 2, 'b', 'b'); fsm.AddEpsilonEdge(0, 2); CompactFSM compact_fsm = fsm.ToCompact(); CompactFSMWithStartEnd compact_fsm_with_start_end(compact_fsm, 0, {false, false, true}); auto json_value = AutoSerializeJSONValue(compact_fsm_with_start_end); ASSERT_TRUE(json_value.is()); // Test literal string comparison - edges are sorted by CompactFSM std::string expected = "[{\"edges\":{\"data_\":[[-1,0,2],[97,97,1],[98,98,2]],\"indptr_\":[0,2,3,3]},\"edge_aux_" "data\":[],\"edge_num\":3},0,[2],false,3]"; ASSERT_EQ(json_value.serialize(), expected); CompactFSMWithStartEnd deserialized; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); // Test basic properties ASSERT_EQ(deserialized.GetFsm().NumStates(), compact_fsm.NumStates()); ASSERT_EQ(deserialized.GetStart(), 0); ASSERT_EQ(deserialized.GetEnds(), std::vector({false, false, true})); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } // Test FSM with rule references { FSM fsm(3); fsm.AddEdge(0, 1, 'a', 'z'); fsm.AddRuleEdge(1, 2, 5); fsm.AddEOSEdge(2, 0); CompactFSM compact_fsm = fsm.ToCompact(); CompactFSMWithStartEnd compact_fsm_with_start_end(compact_fsm, 0, {false, false, true}); auto json_value = AutoSerializeJSONValue(compact_fsm_with_start_end); // Test literal string comparison std::string expected = "[{\"edges\":{\"data_\":[[97,122,1],[-2,5,2],[-3,0,0]],\"indptr_\":[0,1,2,3]},\"edge_aux_" "data\":[],\"edge_num\":3},0,[2],false,3]"; ASSERT_EQ(json_value.serialize(), expected); CompactFSMWithStartEnd deserialized; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_EQ(deserialized.GetFsm().NumStates(), compact_fsm.NumStates()); ASSERT_EQ(deserialized.GetStart(), 0); ASSERT_EQ(deserialized.GetEnds(), std::vector({false, false, true})); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } } TEST(XGrammarSerializationTest, TestComplexStructures) { using namespace xgrammar; // Test vector of FSMEdges { std::vector edges = { {1, 2, 3}, {FSMEdge::EdgeType::kEpsilon, 0, 4}, {FSMEdge::EdgeType::kRuleRef, 5, 6} }; auto json_value = AutoSerializeJSONValue(edges); ASSERT_TRUE(json_value.is()); // Test literal string comparison std::string expected = "[[1,2,3],[-1,0,4],[-2,5,6]]"; ASSERT_EQ(json_value.serialize(), expected); std::vector deserialized; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_EQ(deserialized, edges); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } // Test unordered_map with complex types { std::unordered_map> map = { {"key1", {1, 2, 3}}, {"key2", {4, 5, 6}} }; auto json_value = AutoSerializeJSONValue(map); ASSERT_TRUE(json_value.is()); // Test literal string comparison (note: order might vary in map) // We'll just verify the structure instead of exact string const auto& obj = json_value.get(); ASSERT_EQ(obj.size(), 2); ASSERT_TRUE(obj.find("key1") != obj.end()); ASSERT_TRUE(obj.find("key2") != obj.end()); std::unordered_map> deserialized; auto error = AutoDeserializeJSONValue(&deserialized, json_value); ASSERT_FALSE(error.has_value()); ASSERT_EQ(deserialized, map); // Test roundtrip auto json_value2 = AutoSerializeJSONValue(deserialized); ASSERT_EQ(json_value.serialize(), json_value2.serialize()); } } xgrammar-0.2.3/tests/cpp/test_thread_pool.cc000066400000000000000000000036771521764210300211420ustar00rootroot00000000000000#include #include #include "support/thread_pool.h" using namespace xgrammar; TEST(XGramamrThreadPoolTest, FunctionalTest) { ThreadPool pool(4); // Example 1: Use Submit to submit tasks with return values std::vector> futures; for (int i = 0; i < 8; ++i) { auto fut = pool.Submit([i] { std::this_thread::sleep_for(std::chrono::milliseconds(100)); std::cout << "Task " << i << " is running in thread " << std::this_thread::get_id() << "\n"; return i * i; }); futures.push_back(fut); } for (auto& fut : futures) { int result = fut.get(); std::cout << "Result: " << result << "\n"; } // Example 2: Use Execute to submit tasks without return values for (int i = 0; i < 5; ++i) { pool.Execute([i] { std::this_thread::sleep_for(std::chrono::milliseconds(50)); std::cout << "Execute task " << i << " is running in thread " << std::this_thread::get_id() << "\n"; }); } // Wait for task to complete pool.Join(); } // TEST(XGramamrThreadPoolTest, PressureTest) { // const size_t num_threads = std::thread::hardware_concurrency(); // ThreadPool pool(num_threads); // const size_t num_tasks = 1000; // int counter = 0; // std::mutex counter_mutex; // auto start_time = std::chrono::high_resolution_clock::now(); // for (size_t i = 0; i < num_tasks; ++i) { // pool.Execute([&counter, &counter_mutex, i]() { // std::this_thread::sleep_for(std::chrono::milliseconds(i % 50)); // std::lock_guard lock(counter_mutex); // counter++; // }); // } // pool.Wait(); // auto end_time = std::chrono::high_resolution_clock::now(); // EXPECT_EQ(counter, static_cast(num_tasks)); // auto duration = std::chrono::duration_cast(end_time - start_time); // std::cout << "Pressure test completed, time taken: " << duration.count() << " milliseconds.\n"; // } xgrammar-0.2.3/tests/cpp/test_thread_safe_cache.cc000066400000000000000000000201311521764210300222120ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include #include #include #include "support/logging.h" #include "support/thread_safe_cache.h" using namespace xgrammar; namespace { // static_assert( // sizeof(CompiledGrammar) >= sizeof(std::size_t), // "Our test requires that CompiledGrammar is at least as large as std::size_t" // ); // // simulate a CompiledGrammar object // struct MockGrammar { // std::size_t uuid; // std::byte padding[sizeof(CompiledGrammar) - sizeof(std::size_t)]; // MockGrammar() = default; // MockGrammar(std::size_t uuid) : uuid(uuid) {} // }; // struct SizeEstimator { // template // std::size_t operator()(const T&) const { // return 1; // } // }; // using namespace std::chrono_literals; // struct Computer0 { // inline static auto counter = std::atomic_size_t{}; // inline static constexpr auto kSleepTime = 1000ms; // MockGrammar operator()(std::size_t key) const { // std::this_thread::sleep_for(kSleepTime); // simulate a slow operation // return MockGrammar{counter++}; // } // }; // constexpr auto kUnlimited = std::size_t(-1); // constexpr auto kOverheadRatio = 0.1; // TEST(XGrammarParallelTest, CacheContention) { // XGRAMMAR_LOG_INFO << "Testing the contention performance of the cache (no eviction)"; // constexpr auto kReadGroup = 8; // const auto kNumThreads = int(std::thread::hardware_concurrency()) * 4; // // never evict // auto cache = ThreadSafeLRUCache{kUnlimited}; // auto futures = std::vector>{}; // futures.reserve(kNumThreads); // const auto tic = std::chrono::high_resolution_clock::now(); // const auto target = tic + 1s; // for (int i = 0; i < kNumThreads; ++i) { // futures.push_back(std::async(std::launch::async, [=, &cache] { // std::this_thread::sleep_until(target); // auto sum = std::size_t{}; // // write group: they should not compete with each other // for (int j = 0; j < kNumThreads; ++j) { // sum += cache.Get((i + j) % kNumThreads).uuid; // } // // read group: they should not compete with each other // for (int k = 0; k < kReadGroup; ++k) { // for (int j = 0; j < kNumThreads; ++j) { // sum += cache.Get((i + j) % kNumThreads).uuid; // } // } // return sum; // })); // } // const auto kResult = std::size_t(kNumThreads) * (kNumThreads - 1) / 2 * (1 + kReadGroup); // for (int i = 0; i < kNumThreads; ++i) EXPECT_EQ(futures[i].get(), kResult); // const auto toc = std::chrono::high_resolution_clock::now(); // const auto dur = std::chrono::duration_cast(toc - tic); // // remove 1s sleep time and computing sleep time // const auto overhead = dur - 1s - Computer0::kSleepTime; // XGRAMMAR_LOG_INFO << "(1 write + " << kReadGroup << " reads) " // << "* " << kNumThreads << " threads | " // << "overhead = " << overhead.count() << "ms"; // if (overhead > kOverheadRatio * kNumThreads * Computer0::kSleepTime + 1s) { // XGRAMMAR_LOG(WARNING) << "The overhead is too high, maybe the cache holds the lock too // long?"; // } // } // TEST(XGrammarParallelTest, CacheEviction) { // XGRAMMAR_LOG_INFO << "Testing the eviction performance of the cache (always evict)"; // constexpr auto kInsertGroup = 8; // const auto kNumThreads = int(std::thread::hardware_concurrency()) * 4; // // always evict // auto cache = ThreadSafeLRUCache{0}; // auto futures = std::vector>{}; // futures.reserve(kNumThreads); // const auto tic = std::chrono::high_resolution_clock::now(); // const auto target = tic + 1s; // for (int i = 0; i < kNumThreads; ++i) { // futures.push_back(std::async(std::launch::async, [=, &cache] { // std::this_thread::sleep_until(target); // auto sum = std::size_t{}; // // each thread writes to a different key // for (int j = 0; j < kInsertGroup; ++j) { // sum += cache.Get(i * kInsertGroup + j).uuid; // } // return sum; // })); // } // const auto kNumInsert = std::size_t(kNumThreads) * kInsertGroup; // const auto kResult = kNumInsert * (kNumInsert - 1) / 2; // auto sum = std::size_t{}; // for (int i = 0; i < kNumThreads; ++i) sum += futures[i].get(); // EXPECT_EQ(sum, kResult); // const auto toc = std::chrono::high_resolution_clock::now(); // const auto dur = std::chrono::duration_cast(toc - tic); // // remove 1s sleep time and computing sleep time // const auto overhead = dur - 1s - Computer0::kSleepTime * kInsertGroup; // XGRAMMAR_LOG_INFO << "(" << kInsertGroup << " writes) " // << "* " << kNumThreads << " threads | " // << "overhead = " << overhead.count() << "ms"; // // shouldn't exceed compute + sleep time // if (overhead > kOverheadRatio * Computer0::kSleepTime * kNumThreads + 1s) { // XGRAMMAR_LOG(WARNING) << "The overhead is too high, maybe the cache holds the lock too // long?"; // } // } // // A hook to ensure that the object will not be accessed after its destruction // struct LifeSpanHook { // private: // inline static std::unordered_set manager{}; // inline static std::mutex mutex{}; // static void unsafe_construct(const LifeSpanHook* ptr) { // // insert will return a pair of iterator and bool // EXPECT_TRUE(manager.insert(ptr).second); // } // static void unsafe_destruct(const LifeSpanHook* ptr) { // // erase will return 1 if the element is found and removed // EXPECT_TRUE(manager.erase(ptr)); // } // static void unsafe_confirm(const LifeSpanHook* ptr) { // // ensure that the object is still alive // EXPECT_TRUE(manager.find(ptr) != manager.end()); // } // public: // LifeSpanHook() { // const auto lock = std::lock_guard{mutex}; // unsafe_construct(this); // } // LifeSpanHook(const LifeSpanHook& other) { // const auto lock = std::lock_guard{mutex}; // unsafe_construct(this); // unsafe_confirm(&other); // } // LifeSpanHook& operator=(const LifeSpanHook& other) { // const auto lock = std::lock_guard{mutex}; // unsafe_confirm(this); // unsafe_confirm(&other); // return *this; // } // ~LifeSpanHook() { // const auto lock = std::lock_guard{mutex}; // unsafe_destruct(this); // } // void check() const { // const auto lock = std::lock_guard{mutex}; // unsafe_confirm(this); // } // }; // struct TestObject : LifeSpanHook { // private: // std::string name; // public: // TestObject() = default; // TestObject(std::string name) : name(std::move(name)) {} // TestObject& operator=(std::string name) { // this->check(); // this->name = std::move(name); // return *this; // } // std::string to_string() const { // this->check(); // return this->name; // } // std::size_t MemorySize() const { // this->check(); // return 1; // } // }; // struct Computer1 { // TestObject operator()(const TestObject& key) const { // std::this_thread::sleep_for(5s); // simulate a slow operation // return TestObject{key}; // } // }; // TEST(XGrammarParallelTest, CacheCorrectness) { // auto cache = ThreadSafeLRUCache{kUnlimited}; // const auto kNumThreads = int(std::thread::hardware_concurrency()) * 16; // auto futures = std::vector>{}; // futures.reserve(kNumThreads); // for (auto i = 0; i < kNumThreads; ++i) { // futures.push_back(std::async(std::launch::async, [&cache, i] { // return cache.Get(std::to_string(-i)).to_string(); // })); // } // // Wait the futures to block // std::this_thread::sleep_for(1s); // cache.Clear(); // for (auto i = 0; i < kNumThreads; ++i) { // EXPECT_EQ(futures[i].get(), std::to_string(-i)); // } // } } // namespace xgrammar-0.2.3/tests/cpp/test_utils.h000066400000000000000000000037011521764210300176300ustar00rootroot00000000000000#ifndef XGRAMMAR_TESTS_CPP_TEST_UTILS_H_ #define XGRAMMAR_TESTS_CPP_TEST_UTILS_H_ #include // for ::testing::ContainsRegex #include #include #include /** * @brief Macro to test that a statement throws an exception with a message matching a regex * pattern. * @param statement The statement that should throw an exception. * @param expected_exception The type of exception expected to be thrown. * @param msg_regex Regular expression pattern that the exception message should contain. */ #define XGRAMMAR_EXPECT_THROW(statement, expected_exception, msg_regex) \ EXPECT_THROW( \ { \ try { \ statement; \ } catch (const expected_exception& e) { \ EXPECT_THAT(e.what(), ::testing::ContainsRegex(msg_regex)); \ throw; /* rethrow for EXPECT_THROW to catch */ \ } \ }, \ expected_exception \ ) /** * @brief Macro to test that an std::any value equals an expected value of a specific type. * @param any_val The std::any value to test. * @param type The expected type of the value stored in std::any. * @param val2 The expected value to compare against. */ #define XGRAMMAR_EXPECT_ANY_EQ(any_val, type_name, val2) \ do { \ EXPECT_TRUE(any_val.has_value()); \ EXPECT_TRUE(any_val.type() == typeid(type_name)); \ EXPECT_EQ(std::any_cast(any_val), val2); \ } while (0) #endif // XGRAMMAR_TESTS_CPP_TEST_UTILS_H_ xgrammar-0.2.3/tests/python/000077500000000000000000000000001521764210300160165ustar00rootroot00000000000000xgrammar-0.2.3/tests/python/encoding_dsv32.py000066400000000000000000000362061521764210300212060ustar00rootroot00000000000000# Adapted from https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/main/encoding/encoding_dsv32.py # Used by test_builtin_structural_tag_alignment.py for DeepSeek-V3.2 output extraction. import copy import json import re from typing import Any, Dict, List, Optional, Tuple, Union class DS32EncodingError(Exception): pass TOOLS_SYSTEM_TEMPLATE = """## Tools You have access to a set of tools you can use to answer the user's question. You can invoke functions by writing a "<{dsml_token}function_calls>" block like the following as part of your reply to the user: <{dsml_token}function_calls> <{dsml_token}invoke name="$FUNCTION_NAME"> <{dsml_token}parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE ... <{dsml_token}invoke name="$FUNCTION_NAME2"> ... String and scalar parameters should be specified as is without any escaping or quotes, while lists and objects should use JSON format. The "string" attribute should be set to "true" for string type parameters and "false" for other types (numbers, booleans, arrays, objects). If the thinking_mode is enabled, then after function results you should strongly consider outputting a thinking block. Here is an example: <{dsml_token}function_calls> ... ... {thinking_start_token}...thinking about results{thinking_end_token} Here are the functions available in JSONSchema format: {tool_schemas} """ bos_token: str = "<｜begin▁of▁sentence｜>" eos_token: str = "<｜end▁of▁sentence｜>" thinking_start_token: str = "" thinking_end_token: str = "" dsml_token: str = "｜DSML｜" system_msg_template: str = "{content}" user_msg_template: str = "<｜User｜>{content}<｜Assistant｜>" assistant_msg_template: str = "{reasoning}{content}{tool_calls}<｜end▁of▁sentence｜>" thinking_template = "{reasoning_content}" response_format_template: str = ( "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{schema}" ) tool_call_template: str = '<{dsml_token}invoke name="{name}">\n{arguments}\n' tool_calls_template = "<{dsml_token}function_calls>\n{tool_calls}\n" tool_output_template: str = "\n{content}" def to_json(value: Any) -> str: try: return json.dumps(value, ensure_ascii=False) except Exception: return json.dumps(value, ensure_ascii=True) def tools_from_openai_format(tools): return [tool["function"] for tool in tools] def tool_calls_from_openai_format(tool_calls): return [ {"name": tool_call["function"]["name"], "arguments": tool_call["function"]["arguments"]} for tool_call in tool_calls ] def tool_calls_to_openai_format(tool_calls): return [ { "type": "function", "function": {"name": tool_call["name"], "arguments": tool_call["arguments"]}, } for tool_call in tool_calls ] def encode_arguments_to_dsml(tool_call: Dict[str, str]) -> str: p_dsml_template = ( """<{dsml_token}parameter name="{key}" string="{is_str}">{value}""" ) P_dsml_strs = [] arguments = json.loads(tool_call["arguments"]) for k, v in arguments.items(): p_dsml_str = p_dsml_template.format( dsml_token=dsml_token, key=k, is_str="true" if isinstance(v, str) else "false", value=v if isinstance(v, str) else to_json(v), ) P_dsml_strs.append(p_dsml_str) return "\n".join(P_dsml_strs) def decode_dsml_to_arguments( tool_name: str, tool_args: Dict[str, Tuple[str, str]] ) -> Dict[str, str]: def _decode_value(key: str, value: str, string: str): if string == "true": value = to_json(value) return f"{to_json(key)}: {value}" tool_args_json = ( "{" + ", ".join([_decode_value(k, v, string=is_str) for k, (v, is_str) in tool_args.items()]) + "}" ) return {"name": tool_name, "arguments": tool_args_json} def render_tools(tools: List[Dict[str, Union[str, Dict[str, Any]]]]) -> str: tools_json = [to_json(t) for t in tools] return TOOLS_SYSTEM_TEMPLATE.format( tool_schemas="\n".join(tools_json), dsml_token=dsml_token, thinking_start_token=thinking_start_token, thinking_end_token=thinking_end_token, ) def find_last_user_index(messages: List[Dict[str, Any]]) -> int: last_user_index = -1 for idx in range(len(messages) - 1, -1, -1): if messages[idx].get("role") in ["user", "developer"]: last_user_index = idx break return last_user_index def render_message(index: int, messages: List[Dict[str, Any]], thinking_mode: str) -> str: if not (0 <= index < len(messages)): raise DS32EncodingError( f"Index {index} out of range for messages list of length {len(messages)}" ) if thinking_mode not in ["chat", "thinking"]: raise DS32EncodingError(f"Invalid thinking_mode `{thinking_mode}`") prompt = "" msg = messages[index] last_user_idx = find_last_user_index(messages) role = msg.get("role") content = msg.get("content") tools = msg.get("tools") response_format = msg.get("response_format") tool_calls = msg.get("tool_calls") reasoning_content = msg.get("reasoning_content") if tools: tools = tools_from_openai_format(tools) if tool_calls: tool_calls = tool_calls_from_openai_format(tool_calls) if role == "system": prompt += system_msg_template.format(content=content or "") if tools: prompt += "\n\n" + render_tools(tools) if response_format: prompt += "\n\n" + response_format_template.format(schema=to_json(response_format)) elif role == "developer": if not content: raise DS32EncodingError(f"Invalid message for role `{role}`: {msg}") content_developer = "" if tools: content_developer += "\n\n" + render_tools(tools) if response_format: content_developer += "\n\n" + response_format_template.format( schema=to_json(response_format) ) content_developer += "\n\n# The user's message is: {}".format(content) prompt += user_msg_template.format(content=content_developer) if index == last_user_idx and thinking_mode == "thinking": prompt += thinking_start_token else: prompt += thinking_end_token elif role == "user": prompt += user_msg_template.format(content=content) if index == last_user_idx and thinking_mode == "thinking": prompt += thinking_start_token else: prompt += thinking_end_token elif role == "tool": prev_assistant_idx = index - 1 assistant_msg = messages[prev_assistant_idx] while prev_assistant_idx >= 0 and assistant_msg.get("role") == "tool": prev_assistant_idx -= 1 assistant_msg = messages[prev_assistant_idx] if not ( index == 0 or (prev_assistant_idx >= 0 and assistant_msg.get("role") == "assistant") ): raise DS32EncodingError(f"Invalid messages at {index}:\n{assistant_msg}") tool_call_order = index - prev_assistant_idx assistant_tool_calls = assistant_msg.get("tool_calls") if not (assistant_tool_calls and len(assistant_tool_calls) >= tool_call_order): raise DS32EncodingError("No tool calls but found tool output") if tool_call_order == 1: prompt += "\n\n" prompt += tool_output_template.format(content=content) if tool_call_order == len(assistant_tool_calls): prompt += "\n" if index >= last_user_idx and thinking_mode == "thinking": prompt += "\n\n" + thinking_start_token else: prompt += "\n\n" + thinking_end_token elif role == "assistant": prev_assistant_idx = index thinking_part = "" tool_calls_content = "" if tool_calls: tool_calls = [ tool_call_template.format( dsml_token=dsml_token, name=tool_call.get("name"), arguments=encode_arguments_to_dsml(tool_call), ) for tool_call in tool_calls ] tool_calls_content += "\n\n" + tool_calls_template.format( dsml_token=dsml_token, tool_calls="\n".join(tool_calls) ) summary_content = content or "" if thinking_mode == "thinking" and index > last_user_idx: if not (reasoning_content or tool_calls): raise DS32EncodingError( f"ThinkingMode: {thinking_mode}, invalid message without reasoning_content/tool_calls `{msg}` after last user message" ) thinking_part = ( thinking_template.format(reasoning_content=reasoning_content or "") + thinking_end_token ) prompt += assistant_msg_template.format( reasoning=thinking_part, content=summary_content, tool_calls=tool_calls_content ) else: raise NotImplementedError(f"Unknown role: {role}") return prompt def drop_thinking_messages( messages: List[Dict[str, Any]], last_user_idx: Optional[int] = None ) -> List[Dict[str, Any]]: messages_wo_thinking: List[Dict[str, Any]] = [] last_user_idx = find_last_user_index(messages) if last_user_idx is None else last_user_idx for idx, msg in enumerate(messages): role = msg.get("role") if role in ["user", "system", "tool"] or idx >= last_user_idx: messages_wo_thinking.append(msg) continue elif role == "assistant": msg_wo_thinking = copy.copy(msg) msg_wo_thinking.pop("reasoning_content", None) messages_wo_thinking.append(msg_wo_thinking) return messages_wo_thinking def encode_messages( messages: List[Dict[str, Any]], thinking_mode: str, context: Optional[List[Dict[str, Any]]] = None, drop_thinking: bool = True, add_default_bos_token: bool = True, ) -> str: context = context if context else [] full_messages = context + messages prompt = bos_token if add_default_bos_token and len(context) == 0 else "" if thinking_mode == "thinking" and drop_thinking: full_messages = drop_thinking_messages(full_messages) for idx in range(len(messages)): prompt += render_message(idx + len(context), full_messages, thinking_mode=thinking_mode) return prompt def _read_until_stop(index: int, text: str, stop: List[str]) -> Tuple[int, str, Optional[str]]: min_pos = len(text) matched_stop = None for s in stop: pos = text.find(s, index) if pos != -1 and pos < min_pos: min_pos = pos matched_stop = s if matched_stop: content = text[index:min_pos] return min_pos + len(matched_stop), content, matched_stop else: content = text[index:] return len(text), content, None def parse_tool_calls(index: int, text: str): tool_calls: List[Dict[str, Any]] = [] stop_token = None tool_calls_end_token = f"" while index < len(text): index, _, stop_token = _read_until_stop( index, text, [f"<{dsml_token}invoke", tool_calls_end_token] ) if _ != ">\n": raise DS32EncodingError("Tool call format error") if stop_token == tool_calls_end_token: break if stop_token is None: raise DS32EncodingError("Missing special token") index, tool_name_content, stop_token = _read_until_stop( index, text, [f"<{dsml_token}parameter", f"\n$', tool_name_content, flags=re.DOTALL) if len(p_tool_name) != 1: raise DS32EncodingError("Tool name format error") tool_name = p_tool_name[0] tool_args: Dict[str, Tuple[str, str]] = {} while stop_token == f"<{dsml_token}parameter": index, param_content, stop_token = _read_until_stop( index, text, [f"/{dsml_token}parameter"] ) param_kv = re.findall( r'^ name="(.*?)" string="(true|false)">(.*?)<$', param_content, flags=re.DOTALL ) if len(param_kv) != 1: raise DS32EncodingError("Parameter format error") param_name, string, param_value = param_kv[0] if param_name in tool_args: raise DS32EncodingError("Duplicate parameter name") tool_args[param_name] = (param_value, string) index, content, stop_token = _read_until_stop( index, text, [f"<{dsml_token}parameter", f"\n": raise DS32EncodingError("Parameter format error") tool_call = decode_dsml_to_arguments(tool_name=tool_name, tool_args=tool_args) tool_calls.append(tool_call) return index, stop_token, tool_calls # NOTE: This function is designed to parse only correctly formatted string and will not attempt to correct malformed output that may be generated by the model. def parse_message_from_completion_text(text: str, thinking_mode: str): summary_content, reasoning_content, tool_calls = "", "", [] index, stop_token = 0, None tool_calls_start_token = f"\n\n<{dsml_token}function_calls" is_thinking, is_tool_calling = thinking_mode == "thinking", False if is_thinking: index, content_delta, stop_token = _read_until_stop( index, text, [thinking_end_token, tool_calls_start_token] ) reasoning_content = content_delta if stop_token != thinking_end_token: raise DS32EncodingError("Invalid thinking format") index, content_delta, stop_token = _read_until_stop( index, text, [eos_token, tool_calls_start_token] ) summary_content = content_delta if stop_token == tool_calls_start_token: is_tool_calling = True else: if stop_token != eos_token: raise DS32EncodingError("Invalid summary format") if is_tool_calling: index, stop_token, tool_calls = parse_tool_calls(index, text) index, tool_ends_text, stop_token = _read_until_stop(index, text, [eos_token]) if tool_ends_text: raise DS32EncodingError("Unexpected content after tool calls") if not (len(text) == index and stop_token in [eos_token, None]): raise DS32EncodingError("Unexpected content at end") for sp_token in [bos_token, eos_token, thinking_start_token, thinking_end_token, dsml_token]: if sp_token in summary_content or sp_token in reasoning_content: raise DS32EncodingError("Unexpected special token in content") return { "role": "assistant", "content": summary_content, "reasoning_content": reasoning_content, "tool_calls": tool_calls_to_openai_format(tool_calls), } xgrammar-0.2.3/tests/python/encoding_dsv4.py000066400000000000000000000667621521764210300211370ustar00rootroot00000000000000"""DeepSeek-V4 Encoding Adapted from https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/encoding/encoding_dsv4.py Used by test_builtin_structural_tag_alignment.py for DeepSeek-V4 output extraction. """ import copy import json import re from typing import Any, Dict, List, Optional, Tuple, Union # ============================================================ # Special Tokens # ============================================================ bos_token: str = "<｜begin▁of▁sentence｜>" eos_token: str = "<｜end▁of▁sentence｜>" thinking_start_token: str = "" thinking_end_token: str = "" dsml_token: str = "｜DSML｜" USER_SP_TOKEN = "<｜User｜>" ASSISTANT_SP_TOKEN = "<｜Assistant｜>" LATEST_REMINDER_SP_TOKEN = "<｜latest_reminder｜>" # Task special tokens for internal classification tasks DS_TASK_SP_TOKENS = { "action": "<｜action｜>", "query": "<｜query｜>", "authority": "<｜authority｜>", "domain": "<｜domain｜>", "title": "<｜title｜>", "read_url": "<｜read_url｜>", } VALID_TASKS = set(DS_TASK_SP_TOKENS.keys()) # ============================================================ # Templates # ============================================================ system_msg_template: str = "{content}" user_msg_template: str = "{content}" latest_reminder_msg_template: str = "{content}" assistant_msg_template: str = "{reasoning}{content}{tool_calls}" + eos_token assistant_msg_wo_eos_template: str = "{reasoning}{content}{tool_calls}" thinking_template: str = "{reasoning_content}" response_format_template: str = ( "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{schema}" ) tool_call_template: str = '<{dsml_token}invoke name="{name}">\n{arguments}\n' tool_calls_template = "<{dsml_token}{tc_block_name}>\n{tool_calls}\n" tool_calls_block_name: str = "tool_calls" tool_output_template: str = "{content}" REASONING_EFFORT_MAX = ( "Reasoning Effort: Absolute maximum with no shortcuts permitted.\n" "You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n" "Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n" ) TOOLS_TEMPLATE = """## Tools You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml_token}tool_calls>" block like the following: <{dsml_token}tool_calls> <{dsml_token}invoke name="$TOOL_NAME"> <{dsml_token}parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE ... <{dsml_token}invoke name="$TOOL_NAME2"> ... String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. If thinking_mode is enabled (triggered by {thinking_start_token}), you MUST output your complete reasoning inside {thinking_start_token}...{thinking_end_token} BEFORE any tool calls or final response. Otherwise, output directly after {thinking_end_token} with tool calls or final response. ### Available Tool Schemas {tool_schemas} You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. """ # ============================================================ # Utility Functions # ============================================================ def to_json(value: Any) -> str: """Serialize a value to JSON string.""" try: return json.dumps(value, ensure_ascii=False) except Exception: return json.dumps(value, ensure_ascii=True) def tools_from_openai_format(tools): """Extract function definitions from OpenAI-format tool list.""" return [tool["function"] for tool in tools] def tool_calls_from_openai_format(tool_calls): """Convert OpenAI-format tool calls to internal format.""" return [ {"name": tool_call["function"]["name"], "arguments": tool_call["function"]["arguments"]} for tool_call in tool_calls ] def tool_calls_to_openai_format(tool_calls): """Convert internal tool calls to OpenAI format.""" return [ { "type": "function", "function": {"name": tool_call["name"], "arguments": tool_call["arguments"]}, } for tool_call in tool_calls ] def encode_arguments_to_dsml(tool_call: Dict[str, str]) -> str: """ Encode tool call arguments into DSML parameter format. Args: tool_call: Dict with "name" and "arguments" (JSON string) keys. Returns: DSML-formatted parameter string. """ p_dsml_template = ( '<{dsml_token}parameter name="{key}" string="{is_str}">{value}' ) P_dsml_strs = [] try: arguments = json.loads(tool_call["arguments"]) except Exception: arguments = {"arguments": tool_call["arguments"]} for k, v in arguments.items(): p_dsml_str = p_dsml_template.format( dsml_token=dsml_token, key=k, is_str="true" if isinstance(v, str) else "false", value=v if isinstance(v, str) else to_json(v), ) P_dsml_strs.append(p_dsml_str) return "\n".join(P_dsml_strs) def decode_dsml_to_arguments( tool_name: str, tool_args: Dict[str, Tuple[str, str]] ) -> Dict[str, str]: """ Decode DSML parameters back to a tool call dict. Args: tool_name: Name of the tool. tool_args: Dict mapping param_name -> (value, is_string_flag). Returns: Dict with "name" and "arguments" (JSON string) keys. """ def _decode_value(key: str, value: str, string: str): if string == "true": value = to_json(value) return f"{to_json(key)}: {value}" tool_args_json = ( "{" + ", ".join([_decode_value(k, v, string=is_str) for k, (v, is_str) in tool_args.items()]) + "}" ) return {"name": tool_name, "arguments": tool_args_json} def render_tools(tools: List[Dict[str, Union[str, Dict[str, Any]]]]) -> str: """ Render tool schemas into the system prompt format. Args: tools: List of tool schema dicts (each with name, description, parameters). Returns: Formatted tools section string. """ tools_json = [to_json(t) for t in tools] return TOOLS_TEMPLATE.format( tool_schemas="\n".join(tools_json), dsml_token=dsml_token, thinking_start_token=thinking_start_token, thinking_end_token=thinking_end_token, ) def find_last_user_index(messages: List[Dict[str, Any]]) -> int: """Find the index of the last user/developer message.""" last_user_index = -1 for idx in range(len(messages) - 1, -1, -1): if messages[idx].get("role") in ["user", "developer"]: last_user_index = idx break return last_user_index # ============================================================ # Message Rendering # ============================================================ def render_message( index: int, messages: List[Dict[str, Any]], thinking_mode: str, drop_thinking: bool = True, reasoning_effort: Optional[str] = None, ) -> str: """ Render a single message at the given index into its encoded string form. This is the core function that converts each message in the conversation into the DeepSeek-V4 format. Args: index: Index of the message to render. messages: Full list of messages in the conversation. thinking_mode: Either "chat" or "thinking". drop_thinking: Whether to drop reasoning content from earlier turns. reasoning_effort: Optional reasoning effort level ("max", "high", or None). Returns: Encoded string for this message. """ assert 0 <= index < len(messages) assert thinking_mode in ["chat", "thinking"], f"Invalid thinking_mode `{thinking_mode}`" prompt = "" msg = messages[index] last_user_idx = find_last_user_index(messages) role = msg.get("role") content = msg.get("content") tools = msg.get("tools") response_format = msg.get("response_format") tool_calls = msg.get("tool_calls") reasoning_content = msg.get("reasoning_content") wo_eos = msg.get("wo_eos", False) if tools: tools = tools_from_openai_format(tools) if tool_calls: tool_calls = tool_calls_from_openai_format(tool_calls) # Reasoning effort prefix (only at index 0 in thinking mode with max effort) assert reasoning_effort in [ "max", None, "high", ], f"Invalid reasoning effort: {reasoning_effort}" if index == 0 and thinking_mode == "thinking" and reasoning_effort == "max": prompt += REASONING_EFFORT_MAX if role == "system": prompt += system_msg_template.format(content=content or "") if tools: prompt += "\n\n" + render_tools(tools) if response_format: prompt += "\n\n" + response_format_template.format(schema=to_json(response_format)) elif role == "developer": assert content, f"Invalid message for role `{role}`: {msg}" content_developer = USER_SP_TOKEN content_developer += content if tools: content_developer += "\n\n" + render_tools(tools) if response_format: content_developer += "\n\n" + response_format_template.format( schema=to_json(response_format) ) prompt += user_msg_template.format(content=content_developer) elif role == "user": prompt += USER_SP_TOKEN # Handle content blocks (tool results mixed with text) content_blocks = msg.get("content_blocks") if content_blocks: parts = [] for block in content_blocks: block_type = block.get("type") if block_type == "text": parts.append(block.get("text", "")) elif block_type == "tool_result": tool_content = block.get("content", "") if isinstance(tool_content, list): text_parts = [] for b in tool_content: if b.get("type") == "text": text_parts.append(b.get("text", "")) else: text_parts.append(f"[Unsupported {b.get('type')}]") tool_content = "\n\n".join(text_parts) parts.append(tool_output_template.format(content=tool_content)) else: parts.append(f"[Unsupported {block_type}]") prompt += "\n\n".join(parts) else: prompt += content or "" elif role == "latest_reminder": prompt += LATEST_REMINDER_SP_TOKEN + latest_reminder_msg_template.format(content=content) elif role == "tool": raise NotImplementedError( "deepseek_v4 merges tool messages into user; please preprocess with merge_tool_messages()" ) elif role == "assistant": thinking_part = "" tc_content = "" if tool_calls: tc_list = [ tool_call_template.format( dsml_token=dsml_token, name=tc.get("name"), arguments=encode_arguments_to_dsml(tc), ) for tc in tool_calls ] tc_content += "\n\n" + tool_calls_template.format( dsml_token=dsml_token, tool_calls="\n".join(tc_list), tc_block_name=tool_calls_block_name, ) summary_content = content or "" rc = reasoning_content or "" # Check if previous message has a task - if so, this is a task output (no thinking) prev_has_task = index - 1 >= 0 and messages[index - 1].get("task") is not None if thinking_mode == "thinking" and not prev_has_task: if not drop_thinking or index > last_user_idx: thinking_part = thinking_template.format(reasoning_content=rc) + thinking_end_token else: thinking_part = "" if wo_eos: prompt += assistant_msg_wo_eos_template.format( reasoning=thinking_part, content=summary_content, tool_calls=tc_content ) else: prompt += assistant_msg_template.format( reasoning=thinking_part, content=summary_content, tool_calls=tc_content ) else: raise NotImplementedError(f"Unknown role: {role}") # Append transition tokens based on what follows if index + 1 < len(messages) and messages[index + 1].get("role") not in [ "assistant", "latest_reminder", ]: return prompt task = messages[index].get("task") if task is not None: # Task special token for internal classification tasks assert task in VALID_TASKS, f"Invalid task: '{task}'. Valid tasks are: {list(VALID_TASKS)}" task_sp_token = DS_TASK_SP_TOKENS[task] if task != "action": # Non-action tasks: append task sp token directly after the message prompt += task_sp_token else: # Action task: append Assistant + thinking token + action sp token prompt += ASSISTANT_SP_TOKEN prompt += thinking_end_token if thinking_mode != "thinking" else thinking_start_token prompt += task_sp_token elif messages[index].get("role") in ["user", "developer"]: # Normal generation: append Assistant + thinking token prompt += ASSISTANT_SP_TOKEN if not drop_thinking and thinking_mode == "thinking": prompt += thinking_start_token elif drop_thinking and thinking_mode == "thinking" and index >= last_user_idx: prompt += thinking_start_token else: prompt += thinking_end_token return prompt # ============================================================ # Preprocessing # ============================================================ def merge_tool_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """ Merge tool messages into the preceding user message using content_blocks format. DeepSeek-V4 does not have a standalone "tool" role; instead, tool results are encoded as blocks within user messages. This function converts a standard OpenAI-format conversation (with separate "tool" role messages) into V4 format where tool results are merged into user messages. Args: messages: List of message dicts in OpenAI format. Returns: Processed message list with tool messages merged into user messages. """ merged: List[Dict[str, Any]] = [] for msg in messages: msg = copy.deepcopy(msg) role = msg.get("role") if role == "tool": # Convert tool message to a user message with tool_result block tool_block = { "type": "tool_result", "tool_use_id": msg.get("tool_call_id", ""), "content": msg.get("content", ""), } # Merge into previous message if it's already a user (merged tool) if merged and merged[-1].get("role") == "user" and "content_blocks" in merged[-1]: merged[-1]["content_blocks"].append(tool_block) else: merged.append({"role": "user", "content_blocks": [tool_block]}) elif role == "user": text_block = {"type": "text", "text": msg.get("content", "")} if ( merged and merged[-1].get("role") == "user" and "content_blocks" in merged[-1] and merged[-1].get("task") is None ): merged[-1]["content_blocks"].append(text_block) else: new_msg = { "role": "user", "content": msg.get("content", ""), "content_blocks": [text_block], } # Preserve extra fields (task, wo_eos, mask, etc.) for key in ("task", "wo_eos", "mask"): if key in msg: new_msg[key] = msg[key] merged.append(new_msg) else: merged.append(msg) return merged def sort_tool_results_by_call_order(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """ Sort tool_result blocks within user messages by the order of tool_calls in the preceding assistant message. Args: messages: Preprocessed message list (after merge_tool_messages). Returns: Message list with sorted tool result blocks. """ last_tool_call_order: Dict[str, int] = {} for msg in messages: role = msg.get("role") if role == "assistant" and msg.get("tool_calls"): last_tool_call_order = {} for idx, tc in enumerate(msg["tool_calls"]): tc_id = tc.get("id") or tc.get("function", {}).get("id", "") if tc_id: last_tool_call_order[tc_id] = idx elif role == "user" and msg.get("content_blocks"): tool_blocks = [b for b in msg["content_blocks"] if b.get("type") == "tool_result"] if len(tool_blocks) > 1 and last_tool_call_order: sorted_blocks = sorted( tool_blocks, key=lambda b: last_tool_call_order.get(b.get("tool_use_id", ""), 0) ) sorted_idx = 0 new_blocks = [] for block in msg["content_blocks"]: if block.get("type") == "tool_result": new_blocks.append(sorted_blocks[sorted_idx]) sorted_idx += 1 else: new_blocks.append(block) msg["content_blocks"] = new_blocks return messages # ============================================================ # Main Encoding Function # ============================================================ def encode_messages( messages: List[Dict[str, Any]], thinking_mode: str, context: Optional[List[Dict[str, Any]]] = None, drop_thinking: bool = True, add_default_bos_token: bool = True, reasoning_effort: Optional[str] = None, ) -> str: """ Encode a list of messages into the DeepSeek-V4 prompt format. This is the main entry point for encoding conversations. It handles: - BOS token insertion - Thinking mode with optional reasoning content dropping - Tool message merging into user messages - Multi-turn conversation context Args: messages: List of message dicts to encode. thinking_mode: Either "chat" or "thinking". context: Optional preceding context messages (already encoded prefix). drop_thinking: If True, drop reasoning_content from earlier assistant turns (only keep reasoning for messages after the last user message). add_default_bos_token: Whether to prepend BOS token at conversation start. reasoning_effort: Optional reasoning effort level ("max", "high", or None). Returns: The encoded prompt string. """ context = context if context else [] # Preprocess: merge tool messages and sort tool results messages = merge_tool_messages(messages) messages = sort_tool_results_by_call_order(context + messages)[len(context) :] if context: context = merge_tool_messages(context) context = sort_tool_results_by_call_order(context) full_messages = context + messages prompt = bos_token if add_default_bos_token and len(context) == 0 else "" # Resolve drop_thinking: if any message has tools defined, don't drop thinking effective_drop_thinking = drop_thinking if any(m.get("tools") for m in full_messages): effective_drop_thinking = False if thinking_mode == "thinking" and effective_drop_thinking: full_messages = _drop_thinking_messages(full_messages) # After dropping, recalculate how many messages to render # (context may have shrunk too) num_to_render = len(full_messages) - len(_drop_thinking_messages(context)) context_len = len(full_messages) - num_to_render else: num_to_render = len(messages) context_len = len(context) for idx in range(num_to_render): prompt += render_message( idx + context_len, full_messages, thinking_mode=thinking_mode, drop_thinking=effective_drop_thinking, reasoning_effort=reasoning_effort, ) return prompt def _drop_thinking_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """ Drop reasoning_content and non-essential messages before the last user message. Behavior: - Messages with role in ["user", "system", "tool", "latest_reminder"] are always kept. - Messages at or after the last user index are always kept. - Assistant messages before the last user get reasoning_content removed. - Developer messages before the last user are dropped entirely. """ last_user_idx = find_last_user_index(messages) result = [] keep_roles = {"user", "system", "tool", "latest_reminder", "direct_search_results"} for idx, msg in enumerate(messages): role = msg.get("role") if role in keep_roles or idx >= last_user_idx: result.append(msg) elif role == "assistant": msg = copy.copy(msg) msg.pop("reasoning_content", None) result.append(msg) # developer and other roles before last_user_idx are dropped return result # ============================================================ # Parsing (Decoding model output) # ============================================================ def _read_until_stop(index: int, text: str, stop: List[str]) -> Tuple[int, str, Optional[str]]: """ Read text from index until one of the stop strings is found. Returns: Tuple of (new_index, content_before_stop, matched_stop_string_or_None). """ min_pos = len(text) matched_stop = None for s in stop: pos = text.find(s, index) if pos != -1 and pos < min_pos: min_pos = pos matched_stop = s if matched_stop: content = text[index:min_pos] return min_pos + len(matched_stop), content, matched_stop else: content = text[index:] return len(text), content, None def parse_tool_calls(index: int, text: str) -> Tuple[int, Optional[str], List[Dict[str, str]]]: """ Parse DSML tool calls from text starting at the given index. Args: index: Starting position in text. text: The full text to parse. Returns: Tuple of (new_index, last_stop_token, list_of_tool_call_dicts). Each tool call dict has "name" and "arguments" keys. """ tool_calls: List[Dict[str, Any]] = [] stop_token = None tool_calls_end_token = f"" while index < len(text): index, _, stop_token = _read_until_stop( index, text, [f"<{dsml_token}invoke", tool_calls_end_token] ) if _ != ">\n": raise ValueError(f"Tool call format error: expected '>\\n' but got '{_}'") if stop_token == tool_calls_end_token: break if stop_token is None: raise ValueError("Missing special token in tool calls") index, tool_name_content, stop_token = _read_until_stop( index, text, [f"<{dsml_token}parameter", f"\n$', tool_name_content, flags=re.DOTALL) if len(p_tool_name) != 1: raise ValueError(f"Tool name format error: '{tool_name_content}'") tool_name = p_tool_name[0] tool_args: Dict[str, Tuple[str, str]] = {} while stop_token == f"<{dsml_token}parameter": index, param_content, stop_token = _read_until_stop( index, text, [f"/{dsml_token}parameter"] ) param_kv = re.findall( r'^ name="(.*?)" string="(true|false)">(.*?)<$', param_content, flags=re.DOTALL ) if len(param_kv) != 1: raise ValueError(f"Parameter format error: '{param_content}'") param_name, string, param_value = param_kv[0] if param_name in tool_args: raise ValueError(f"Duplicate parameter name: '{param_name}'") tool_args[param_name] = (param_value, string) index, content, stop_token = _read_until_stop( index, text, [f"<{dsml_token}parameter", f"\n": raise ValueError(f"Parameter format error: expected '>\\n' but got '{content}'") tool_call = decode_dsml_to_arguments(tool_name=tool_name, tool_args=tool_args) tool_calls.append(tool_call) return index, stop_token, tool_calls def parse_message_from_completion_text(text: str, thinking_mode: str) -> Dict[str, Any]: """ Parse a model completion text into a structured assistant message. This function takes the raw text output from the model (a single assistant turn) and extracts: - reasoning_content (thinking block) - content (summary/response) - tool_calls (if any) NOTE: This function is designed to parse only correctly formatted strings and will raise ValueError for malformed output. Args: text: The raw completion text (including EOS token). thinking_mode: Either "chat" or "thinking". Returns: Dict with keys: "role", "content", "reasoning_content", "tool_calls". tool_calls are in OpenAI format. """ summary_content, reasoning_content, tool_calls = "", "", [] index, stop_token = 0, None tool_calls_start_token = f"\n\n<{dsml_token}{tool_calls_block_name}" is_thinking = thinking_mode == "thinking" is_tool_calling = False if is_thinking: index, content_delta, stop_token = _read_until_stop( index, text, [thinking_end_token, tool_calls_start_token] ) reasoning_content = content_delta assert stop_token == thinking_end_token, "Invalid thinking format: missing " index, content_delta, stop_token = _read_until_stop( index, text, [eos_token, tool_calls_start_token] ) summary_content = content_delta if stop_token == tool_calls_start_token: is_tool_calling = True else: assert stop_token == eos_token, "Invalid format: missing EOS token" if is_tool_calling: index, stop_token, tool_calls = parse_tool_calls(index, text) index, tool_ends_text, stop_token = _read_until_stop(index, text, [eos_token]) assert not tool_ends_text, "Unexpected content after tool calls" assert len(text) == index and stop_token in [eos_token, None], "Unexpected content at end" for sp_token in [bos_token, eos_token, thinking_start_token, thinking_end_token, dsml_token]: assert ( sp_token not in summary_content and sp_token not in reasoning_content ), f"Unexpected special token '{sp_token}' in content" return { "role": "assistant", "content": summary_content, "reasoning_content": reasoning_content, "tool_calls": tool_calls_to_openai_format(tool_calls), } xgrammar-0.2.3/tests/python/test_builtin_structural_tag.py000066400000000000000000001433731521764210300242330ustar00rootroot00000000000000"""Tests for get_structural_tag_for_model and generated structural tags.""" import re import time from typing import Any, Dict, List, Optional, Tuple import pytest from transformers import AutoTokenizer import xgrammar as xgr from xgrammar.builtin_structural_tag import ( get_deepseek_r1_structural_tag, get_deepseek_v3_1_structural_tag, get_deepseek_v3_2_structural_tag, get_deepseek_v4_structural_tag, get_glm_4_7_structural_tag, get_harmony_structural_tag, get_kimi_structural_tag, get_llama_structural_tag, get_minimax_structural_tag, get_model_structural_tag, get_qwen_3_5_structural_tag, get_qwen_3_coder_structural_tag, get_qwen_3_structural_tag, normalize_tool_choice, ) from xgrammar.openai_tool_call_schema import BuiltinToolParam, FunctionToolParam from xgrammar.structural_tag import JSONSchemaFormat, StructuralTag, TagFormat from xgrammar.testing import _is_grammar_accept_string def _input_dict_to_get_stag_kwargs(format_type: str, input_dict: Dict[str, Any]) -> Dict[str, Any]: """Convert input_dict (used by old template function API) to kwargs for get_structural_tag_for_model.""" tools = input_dict.get("tools", []) if isinstance(tools, list): tools = list(tools) builtin_tools = input_dict.get("builtin_tools", []) if not isinstance(builtin_tools, list): tools = builtin_tools builtin_tools = [] for builtin_tool in builtin_tools: function = builtin_tool.get("function", {}) tools.append( { "type": function.get("name"), "name": function.get("name"), "parameters": function.get("parameters"), } ) tool_choice = input_dict.get("tool_choice", "auto") if tool_choice == "forced": tool_choice = { "type": "function", "function": {"name": input_dict.get("forced_function_name")}, } return { "model": format_type, "tools": tools, "reasoning": input_dict.get("reasoning", input_dict.get("reasoning", True)), "tool_choice": tool_choice, } # ---------- Fixtures / Helpers ---------- class Profiler: def __init__(self, tokenizer_id: str): tokenizer = AutoTokenizer.from_pretrained( tokenizer_id, use_fast=True, trust_remote_code=True ) self.tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) self.compiler = xgr.GrammarCompiler( self.tokenizer_info, max_threads=16, cache_enabled=False ) def profile_stag(self, structural_tag: StructuralTag, instance: str): time_begin = time.monotonic_ns() compiled_grammar = self.compiler.compile_structural_tag(structural_tag) time_end = time.monotonic_ns() compiler_duration = time_end - time_begin print(f"Compiling structural tag {structural_tag.format}") print(f"Compile time: {compiler_duration / 1000 / 1000} ms") matcher = xgr.GrammarMatcher(compiled_grammar) token_bitmask = xgr.allocate_token_bitmask(1, self.tokenizer_info.vocab_size) print(f"Matching instance: {instance}") for char in instance: matcher.accept_string(char) time_begin = time.monotonic_ns() matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() duration = time_end - time_begin print(f"Time to generate mask: {duration / 1000} us, Character: '{char}'") profiler: Optional[Profiler] = None PROFILER_ON = True tokenizer_id = "meta-llama/Llama-3.1-8B-Instruct" @pytest.fixture(autouse=True, scope="module") def disable_profiler(request): global PROFILER_ON global profiler # Import shared token check from conftest (handles env vars + cached login) from conftest import _hf_token_available, _hf_token_explicitly_disabled if not _hf_token_available() or _hf_token_explicitly_disabled(request.config): PROFILER_ON = False else: profiler = Profiler(tokenizer_id) def check_stag_with_instance( structural_tag: StructuralTag, instance: str, is_accepted: bool = True, debug_print: bool = False, ): stag_grammar = xgr.Grammar.from_structural_tag(structural_tag) accepted = _is_grammar_accept_string(stag_grammar, instance, debug_print=debug_print) assert accepted == is_accepted if PROFILER_ON: profiler.profile_stag(structural_tag, instance) def _walk_structural_format(format_obj): """Yield every nested structural format object.""" yield format_obj for attr_name in ("content", "format"): child = getattr(format_obj, attr_name, None) if child is not None: yield from _walk_structural_format(child) for attr_name in ("elements", "tags"): for child in getattr(format_obj, attr_name, []) or []: yield from _walk_structural_format(child) def _collect_tag_begins(structural_tag: StructuralTag) -> List[str]: """Collect TagFormat begin strings from a structural tag.""" return [ format_obj.begin for format_obj in _walk_structural_format(structural_tag.format) if isinstance(format_obj, TagFormat) and isinstance(format_obj.begin, str) ] def _collect_json_schema_values(structural_tag: StructuralTag) -> List[Any]: """Collect JSON schema values from nested JSONSchemaFormat nodes.""" return [ format_obj.json_schema for format_obj in _walk_structural_format(structural_tag.format) if isinstance(format_obj, JSONSchemaFormat) ] def _collect_excludes(structural_tag: StructuralTag) -> List[List[str]]: """Collect every ``excludes`` list from nested AnyText / TriggeredTags nodes.""" return [ list(format_obj.excludes) for format_obj in _walk_structural_format(structural_tag.format) if getattr(format_obj, "excludes", None) is not None ] # ---------- Shared tool definitions ---------- SIMPLE_SCHEMA = {"type": "object", "properties": {"q": {"type": "string"}}} def make_tools(names: List[str], schema: Dict[str, Any] = SIMPLE_SCHEMA) -> List[Dict[str, Any]]: return [{"function": {"name": n, "parameters": schema}} for n in names] # Tool lists used by instance tests (all in one place) _tools_llama = make_tools(["t1"]) _tools_kimi = make_tools(["get_weather"]) _tools_deepseek = make_tools(["search"]) _tools_qwen_3_coder = make_tools(["run_sql"]) _tools_qwen_3 = make_tools(["t1"]) _tools_qwen_3_5 = make_tools(["run_sql"]) _tools_harmony = make_tools(["comment_tool"]) _builtin_harmony = make_tools(["analysis_tool"]) _tools_deepseek_v3_2 = make_tools(["search"]) _tools_deepseek_v4 = make_tools(["search"]) _tools_minimax = make_tools(["search"]) _tools_glm_4_7 = make_tools(["search"]) # Two distinct tools for tool_choice=required / forced instance tests. _tools_llama_pair = make_tools(["t1", "t2"]) _tools_kimi_pair = make_tools(["t1", "t2"]) _tools_deepseek_pair = make_tools(["search", "alt"]) _tools_deepseek_v3_2_pair = make_tools(["search", "alt"]) _tools_deepseek_v4_pair = make_tools(["search", "alt"]) _tools_minimax_pair = make_tools(["search", "alt"]) _tools_qwen_3_coder_pair = make_tools(["run_sql", "run_py"]) _tools_qwen_3_pair = make_tools(["t1", "t2"]) _tools_qwen_3_5_pair = make_tools(["run_sql", "run_py"]) _tools_harmony_pair = make_tools(["comment_tool", "other_tool"]) _tools_glm_4_7_pair = make_tools(["search", "alt"]) # ---------- Test: unknown format type ---------- def test_unknown_format(): """get_structural_tag_for_model raises ValueError for unknown format type.""" with pytest.raises(ValueError) as exc_info: get_model_structural_tag("unknown_format") assert "Unknown format type" in str(exc_info.value) assert "unknown_format" in str(exc_info.value) def test_unknown_format_is_checked_before_tool_inputs(): """Unknown model names are rejected before validating tool inputs.""" with pytest.raises(ValueError) as exc_info: get_model_structural_tag("unknown_format", tools="not_a_list") assert "Unknown format type" in str(exc_info.value) assert "unknown_format" in str(exc_info.value) # ---------- Test: input validation errors ---------- # (format_type, input_dict, substring that must appear in the error message) input_validation_error_cases: List[Tuple[str, Dict[str, Any], str]] = [ # tools must be a list ("llama", {"tools": "not_a_list"}, "must be a list"), ("llama", {"tools": 123}, "must be a list"), # tool[function] must have "name" and "parameters" ("llama", {"tools": [{"function": {}}]}, "function.name"), ("llama", {"tools": [{"function": {"parameters": {}}}]}, "function.name"), # name must be string ("llama", {"tools": [{"function": {"name": 123, "parameters": {}}}]}, "function.name"), # parameters must be dict ( "llama", {"tools": [{"function": {"name": "t1", "parameters": "not_a_dict"}}]}, "function.parameters", ), ("llama", {"tools": [{"function": {"name": "t1", "parameters": []}}]}, "function.parameters"), # Legacy builtin_tools test data is converted into public tools before calling the API. ("harmony", {"tools": [], "builtin_tools": "not_list"}, "must be a list"), ( "harmony", {"tools": [], "builtin_tools": [{"function": {"name": "b1", "parameters": 1}}]}, "parameters", ), ] @pytest.mark.parametrize("format_type, input_dict, error_substring", input_validation_error_cases) def test_input_validation_errors( format_type: str, input_dict: Dict[str, Any], error_substring: str ): """get_model_structural_tag raises ValueError for invalid input.""" with pytest.raises(ValueError) as exc_info: get_model_structural_tag(**_input_dict_to_get_stag_kwargs(format_type, input_dict)) msg = str(exc_info.value) if ".*" in error_substring: assert re.search( error_substring, msg, re.DOTALL ), f"Expected match for {error_substring!r} in {msg!r}" else: assert error_substring in msg, f"Expected {error_substring!r} in {msg!r}" @pytest.mark.parametrize( "kwargs, error_substring", [ # Required mode needs at least one function or builtin tool after filtering. ({"tools": [], "tool_choice": "required"}, "required"), # Named function choices must reference an existing function tool. ( { "tools": make_tools(["get_weather"]), "tool_choice": {"type": "function", "function": {"name": "missing"}}, }, "missing", ), # Builtin choices must reference an existing builtin tool type. ( { "tools": [ { "type": "web_search_preview", "name": "browser.search", "parameters": SIMPLE_SCHEMA, } ], "tool_choice": {"type": "code_interpreter"}, }, "exactly one", ), # Builtin choices by type are ambiguous when multiple builtin tools share a type. ( { "tools": [ { "type": "web_search_preview", "name": "browser.search", "parameters": SIMPLE_SCHEMA, }, { "type": "web_search_preview", "name": "browser.open", "parameters": SIMPLE_SCHEMA, }, ], "tool_choice": {"type": "web_search_preview"}, }, "exactly one", ), # Allowed tool refs must reference available builtin tools. ( { "tools": make_tools(["get_weather"]), "tool_choice": { "type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": [{"type": "web_search_preview"}]}, }, }, "Allowed builtin", ), ], ) def test_public_api_validation_errors(kwargs: Dict[str, Any], error_substring: str): """Public API rejects invalid tool and tool_choice combinations.""" with pytest.raises(ValueError) as exc_info: get_model_structural_tag("harmony", **kwargs) assert error_substring in str(exc_info.value) def test_normalize_tool_choice_named_function(): """normalize_tool_choice returns one forced function for named choices.""" function_tools, builtin_tools, simplified_tool_choice = normalize_tool_choice( tools=make_tools(["get_weather", "get_time"]), tool_choice={"type": "function", "function": {"name": "get_weather"}}, ) assert [tool.function.name for tool in function_tools] == ["get_weather"] assert builtin_tools == [] assert simplified_tool_choice == "forced" def test_normalize_tool_choice_allowed_tools(): """normalize_tool_choice filters function and builtin tools for allowed_tools.""" function_tools, builtin_tools, simplified_tool_choice = normalize_tool_choice( tools=[ *make_tools(["get_weather", "get_time"]), {"type": "web_search_preview", "name": "browser.search", "parameters": SIMPLE_SCHEMA}, {"type": "code_interpreter", "name": "browser.open", "parameters": SIMPLE_SCHEMA}, ], tool_choice={ "type": "allowed_tools", "allowed_tools": { "mode": "required", "tools": [ {"type": "function", "function": {"name": "get_weather"}}, {"type": "web_search_preview"}, ], }, }, ) assert [tool.function.name for tool in function_tools] == ["get_weather"] assert [tool.name for tool in builtin_tools] == ["browser.search"] assert simplified_tool_choice == "required" def test_normalize_tool_choice_none_clears_tools(): """normalize_tool_choice maps public none to text-only auto.""" function_tools, builtin_tools, simplified_tool_choice = normalize_tool_choice( tools=[ *make_tools(["get_weather"]), {"type": "web_search_preview", "name": "browser.search", "parameters": SIMPLE_SCHEMA}, ], tool_choice="none", ) assert function_tools == [] assert builtin_tools == [] assert simplified_tool_choice == "auto" def test_public_tool_shapes(): """Public tools accepts dict, FunctionToolParam, and BuiltinToolParam values.""" structural_tag = get_model_structural_tag( "harmony", tools=[ {"type": "function", "function": {"name": "get_weather", "parameters": SIMPLE_SCHEMA}}, FunctionToolParam( function={"name": "get_time", "parameters": {"type": "object", "properties": {}}} ), BuiltinToolParam( type="web_search_preview", name="browser.search", parameters={"type": "object", "properties": {"query": {"type": "string"}}}, ), ], reasoning=False, ) begins = _collect_tag_begins(structural_tag) assert any("get_weather" in begin for begin in begins) assert any("get_time" in begin for begin in begins) assert any("browser.search" in begin for begin in begins) assert xgr.Grammar.from_structural_tag(structural_tag) is not None def test_named_choice_forces_function(): """Named function tool_choice is normalized to one forced function tool.""" structural_tag = get_model_structural_tag( "llama", tools=make_tools(["get_weather", "get_time"]), tool_choice={"type": "function", "function": {"name": "get_weather"}}, reasoning=False, ) begins = _collect_tag_begins(structural_tag) assert any("get_weather" in begin for begin in begins) assert not any("get_time" in begin for begin in begins) def test_builtin_choice_forces_builtin(): """Builtin tool_choice is normalized to one forced builtin tool.""" structural_tag = get_model_structural_tag( "harmony", tools=[ *make_tools(["get_weather"]), { "type": "web_search_preview", "name": "browser.search", "parameters": {"type": "object", "properties": {"query": {"type": "string"}}}, }, { "type": "code_interpreter", "name": "browser.open", "parameters": {"type": "object", "properties": {"url": {"type": "string"}}}, }, ], tool_choice={"type": "web_search_preview"}, reasoning=False, ) begins = _collect_tag_begins(structural_tag) assert any("browser.search" in begin for begin in begins) assert not any("browser.open" in begin for begin in begins) assert not any("get_weather" in begin for begin in begins) def test_allowed_choice_filters_tools(): """Allowed tools filters both function tools and builtin tools.""" structural_tag = get_model_structural_tag( "harmony", tools=[ *make_tools(["get_weather", "get_time"]), { "type": "web_search_preview", "name": "browser.search", "parameters": {"type": "object", "properties": {"query": {"type": "string"}}}, }, { "type": "code_interpreter", "name": "browser.open", "parameters": {"type": "object", "properties": {"url": {"type": "string"}}}, }, ], tool_choice={ "type": "allowed_tools", "allowed_tools": { "mode": "auto", "tools": [ {"type": "function", "function": {"name": "get_weather"}}, {"type": "web_search_preview"}, ], }, }, reasoning=False, ) begins = _collect_tag_begins(structural_tag) assert any("get_weather" in begin for begin in begins) assert any("browser.search" in begin for begin in begins) assert not any("get_time" in begin for begin in begins) assert not any("browser.open" in begin for begin in begins) def test_none_parameters_unconstrained(): """None parameters are converted to the True JSON schema.""" structural_tag = get_model_structural_tag( "llama", tools=[{"type": "function", "function": {"name": "ping", "parameters": None}}], reasoning=False, ) json_schema_values = _collect_json_schema_values(structural_tag) assert True in json_schema_values assert None not in json_schema_values def test_harmony_builtin_tool_instance(): """Harmony builtin tools follow the browser sample shape from harmony examples.""" structural_tag = get_model_structural_tag( "harmony", tools=[ { "type": "web_search_preview", "name": "browser.search", "parameters": { "type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"], }, } ], reasoning=False, ) check_stag_with_instance( structural_tag, '<|channel|>commentary to=browser.search code<|message|>{"query": "weather"}<|call|>', True, ) check_stag_with_instance( structural_tag, '<|channel|>analysis to=browser.search<|message|>{"query": "weather"}<|call|>', False, ) def test_kimi_auto_requires_tool_calls_section(): """Kimi auto tool calls must use the official section wrapper.""" # Excluding special tokens in free text is what forbids a bare # <|tool_call_begin|> outside the <|tool_calls_section_begin|> wrapper. It is # on by default; passed explicitly here to make the dependency clear. structural_tag = get_model_structural_tag( "kimi", tools=_tools_kimi, reasoning=False, exclude_special_tokens=True ) assert "<|tool_calls_section_begin|>" in structural_tag.model_dump_json() check_stag_with_instance( structural_tag, '<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{"q": "v"}<|tool_call_end|><|tool_calls_section_end|>', True, ) check_stag_with_instance( structural_tag, '<|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{"q": "v"}<|tool_call_end|>', False, ) @pytest.mark.parametrize( "structural_tag_fn", [ get_llama_structural_tag, get_kimi_structural_tag, get_deepseek_r1_structural_tag, get_deepseek_v3_1_structural_tag, get_qwen_3_5_structural_tag, get_qwen_3_coder_structural_tag, get_qwen_3_structural_tag, get_harmony_structural_tag, get_deepseek_v3_2_structural_tag, get_deepseek_v4_structural_tag, get_minimax_structural_tag, get_glm_4_7_structural_tag, ], ) @pytest.mark.parametrize( "case", [ # Normal case: one function tool and one builtin tool are both available. { "tools": [ FunctionToolParam( function={ "name": "get_weather", "parameters": { "type": "object", "properties": {"location": {"type": "string"}}, }, } ) ], "builtin_tools": [ BuiltinToolParam( type="web_search_preview", name="browser.search", parameters={"type": "object", "properties": {"query": {"type": "string"}}}, ) ], "tool_choice": "auto", }, # Empty auto case: public "none" is normalized to no tools plus "auto". {"tools": [], "builtin_tools": [], "tool_choice": "auto"}, # None parameters case: missing schema must become unconstrained JSON. { "tools": [FunctionToolParam(function={"name": "ping", "parameters": None})], "builtin_tools": [], "tool_choice": "auto", }, # Non-strict case: strict=False ignores the provided schema. { "tools": [ FunctionToolParam( function={ "name": "ping", "parameters": { "type": "object", "properties": {"message": {"type": "string"}}, }, "strict": False, } ) ], "builtin_tools": [], "tool_choice": "auto", }, # Forced case: the top-level API has already filtered to one tool. { "tools": [FunctionToolParam(function={"name": "ping", "parameters": None})], "builtin_tools": [], "tool_choice": "forced", }, ], ) def test_specific_functions_cases(structural_tag_fn, case: Dict[str, Any]): """Specific functions accept normalized internal inputs from the public API.""" structural_tag = structural_tag_fn( tools=case["tools"], builtin_tools=case["builtin_tools"], tool_choice=case["tool_choice"], reasoning=True, ) assert isinstance(structural_tag, StructuralTag) xgr.Grammar.from_structural_tag(structural_tag) assert None not in _collect_json_schema_values(structural_tag) # ---------- Test: exclude_special_tokens ---------- # Model keys whose built-in structural tags forbid special tokens (e.g. ) # from appearing in free-text spans when the exclusion is enabled. Harmony has no # such excludes and is covered separately. _EXCLUDE_TOKEN_MODELS = [ "llama", "kimi", "deepseek_r1", "deepseek_v3_1", "deepseek_v3_2", "deepseek_v4", "qwen_3", "qwen_3_5", "minimax", "glm_4_7", ] @pytest.mark.parametrize("model", _EXCLUDE_TOKEN_MODELS) # Tools present -> TriggeredTagsFormat excludes; no tools -> AnyTextFormat excludes. @pytest.mark.parametrize("tools", [make_tools(["search"]), []]) def test_exclude_special_tokens_default_excludes_think_tokens(model, tools): """By default the built-in structural tag excludes the special tokens from free text.""" structural_tag = get_model_structural_tag(model, tools=tools, reasoning=True) flat = [token for excludes in _collect_excludes(structural_tag) for token in excludes] assert "" in flat assert "" in flat xgr.Grammar.from_structural_tag(structural_tag) @pytest.mark.parametrize("model", _EXCLUDE_TOKEN_MODELS) @pytest.mark.parametrize("tools", [make_tools(["search"]), []]) def test_exclude_special_tokens_false_excludes_nothing(model, tools): """Opting out with ``exclude_special_tokens=False`` excludes nothing from free text.""" structural_tag = get_model_structural_tag( model, tools=tools, reasoning=True, exclude_special_tokens=False ) assert all(excludes == [] for excludes in _collect_excludes(structural_tag)) # The less-restrictive grammar must still build. xgr.Grammar.from_structural_tag(structural_tag) @pytest.mark.parametrize("flag", [False, True]) def test_exclude_special_tokens_harmony_is_no_op(flag): """Harmony has no special tokens to exclude, so the flag never adds excludes.""" structural_tag = get_model_structural_tag( "harmony", tools=make_tools(["search"]), exclude_special_tokens=flag ) assert all(excludes == [] for excludes in _collect_excludes(structural_tag)) def test_exclude_special_tokens_passed_to_specific_function(): """The flag also reaches the model-specific builders directly via kwargs.""" tools = [FunctionToolParam(function={"name": "search", "parameters": SIMPLE_SCHEMA})] off = get_qwen_3_structural_tag(tools=tools, exclude_special_tokens=False) assert all(excludes == [] for excludes in _collect_excludes(off)) on = get_qwen_3_structural_tag(tools=tools, exclude_special_tokens=True) flat = [token for excludes in _collect_excludes(on) for token in excludes] assert "" in flat assert "" in flat @pytest.mark.parametrize( "format_type, instance, is_accepted", [ ("llama", '{"name": "t1", "parameters": {"q": "v"}}', True), ( "kimi", '123<|tool_calls_section_begin|><|tool_call_begin|>functions.t1:0<|tool_call_argument_begin|>{"q": "v"}<|tool_call_end|><|tool_calls_section_end|>', True, ), ( "kimi", '<|tool_call_begin|>functions.t1:0<|tool_call_argument_begin|>{"q": "v"}<|tool_call_end|>', False, ), ( "kimi", '123<|tool_calls_section_begin|><|tool_call_begin|>functions.t2:0<|tool_call_argument_begin|>{"q": "v"}<|tool_call_end|><|tool_calls_section_end|>', False, ), ( "deepseek_r1", 'text<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>t1\n```json\n{"q": "v"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>', True, ), ( "deepseek_r1", 'text<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>t2\n```json\n{"q": "v"}\n```<｜tool▁call▁end｜>', False, ), ( "deepseek_v3_2", '<｜DSML｜function_calls>\n<｜DSML｜invoke name="t1">\n<｜DSML｜parameter name="q" string="false">{"type": "string"}\n\n\n', True, ), ( "deepseek_v3_2", '<｜DSML｜function_calls>\n<｜DSML｜invoke name="t2">\n<｜DSML｜parameter name="q" string="false">{"type": "string"}\n\n\n', False, ), ( "deepseek_v4", '<｜DSML｜tool_calls>\n<｜DSML｜invoke name="t1">\n<｜DSML｜parameter name="q" string="false">{"type": "string"}\n\n\n', True, ), ( "deepseek_v4", '<｜DSML｜tool_calls>\n<｜DSML｜invoke name="t2">\n<｜DSML｜parameter name="q" string="false">{"type": "string"}\n\n\n', False, ), ( "minimax", '\n\n\n\n\n\n\n{"type": "string"}\n\n\n', True, ), ( "minimax", '\n\n{"type": "string"}\n\n\n', False, ), ( "qwen_3_coder", '\n\n{"type": "string"}\n\n', True, ), ( "qwen_3_5", '\n\n{"type": "string"}\n\n', True, ), ("qwen_3", 'text\n{"name": "t1", "arguments": {"q": "v"}}\n', True), ("qwen_3", 'text\n{"name": "t2", "arguments": {"q": "v"}}\n', False), ("qwen_3", 'text\n{"name": "t1", "arguments": {"q": "v"}}\n', True), ("qwen_3", 'text\n{"name": "t2", "arguments": {"q": "v"}}\n', False), ( "harmony", '<|channel|>commentary to=functions.t1<|constrain|>json<|message|>{"q": "v"}<|call|>', True, ), ( "harmony", '<|channel|>commentary to=functions.t2<|constrain|>json<|message|>{"q": "v"}<|call|>', False, ), ], ) @pytest.mark.parametrize( "tool", [ { "function": { "name": "t1", "strict": False, "parameters": {"type": "object", "properties": {"q": {"type": "string"}}}, } }, # strict=False without parameters {"function": {"name": "t1", "strict": False}}, # no strict, no parameters {"function": {"name": "t1"}}, ], ) def test_strict_or_missing_parameters( format_type: str, instance: str, is_accepted: bool, tool: Dict[str, Any] ): """strict=False or missing 'parameters' should still accept/reject instances correctly.""" tools = [tool] # Special-token exclusion (on by default) is what makes markers appearing # outside their required wrapper (e.g. a bare Kimi <|tool_call_begin|> # without the tool-calls section) get rejected; passed explicitly here to # make the dependency clear. stag = get_model_structural_tag( format_type, tools=tools, reasoning=False, exclude_special_tokens=True ) check_stag_with_instance(stag, instance, is_accepted) # ---------- Test: instance positive / negative ---------- # Case: (input_dict, instances, reasoning, expected_grammar_ebnf, expected_accept_per_instance) # input_dict may include tool_choice ("auto" | "forced" | "required") and forced_function_name (str | None). # When expected_grammar_ebnf is empty or whitespace-only, grammar equality is skipped (fill in EBNF when ready). InstanceCase = Tuple[Dict[str, Any], List[str], bool, List[bool]] def run_instance_case(format_type: str, case: InstanceCase): """Run one instance test case (accept/reject per instance string).""" (input_dict, instances, reasoning, expected_accept_per_instance) = case kwargs = _input_dict_to_get_stag_kwargs(format_type, input_dict) kwargs["reasoning"] = reasoning stag = get_model_structural_tag(**kwargs) for j, instance in enumerate(instances): check_stag_with_instance(stag, instance, expected_accept_per_instance[j]) # tool_choice=required / forced: expected_grammar_ebnf left "" for manual completion. _tool_choice_instance_cases = [ pytest.param( "llama", ( {"tools": _tools_llama_pair, "tool_choice": "required"}, ["", '{"name": "t1", "parameters": {"q": "v"}}'], False, [False, True], ), id="llama-required", ), pytest.param( "llama", ( {"tools": _tools_llama_pair, "tool_choice": "forced", "forced_function_name": "t1"}, [ '{"name": "t1", "parameters": {"q": "v"}}', '{"name": "t2", "parameters": {"q": "v"}}', ], False, [True, False], ), id="llama-forced", ), pytest.param( "kimi", ( {"tools": _tools_kimi_pair, "tool_choice": "required"}, [ "", '<|tool_calls_section_begin|><|tool_call_begin|>functions.t1:0<|tool_call_argument_begin|>{"q": "v"}<|tool_call_end|><|tool_calls_section_end|>', ], False, [False, True], ), id="kimi-required", ), pytest.param( "kimi", ( {"tools": _tools_kimi_pair, "tool_choice": "forced", "forced_function_name": "t1"}, [ '<|tool_calls_section_begin|><|tool_call_begin|>functions.t1:0<|tool_call_argument_begin|>{"q": "v"}<|tool_call_end|><|tool_calls_section_end|>', '<|tool_calls_section_begin|><|tool_call_begin|>functions.t2:0<|tool_call_argument_begin|>{"q": "v"}<|tool_call_end|><|tool_calls_section_end|>', ], False, [True, False], ), id="kimi-forced", ), pytest.param( "deepseek_r1", ( {"tools": _tools_deepseek_pair, "tool_choice": "required"}, [ "", '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>search\n```json\n{"q": "v"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>', ], False, [False, True], ), id="deepseek_r1-required", ), pytest.param( "deepseek_r1", ( { "tools": _tools_deepseek_pair, "tool_choice": "forced", "forced_function_name": "search", }, [ '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>search\n```json\n{"q": "v"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>', '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>alt\n```json\n{"q": "v"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>', ], False, [True, False], ), id="deepseek_r1-forced", ), pytest.param( "deepseek_v3_2", ( {"tools": _tools_deepseek_v3_2_pair, "tool_choice": "required"}, [ "", '\n\n<｜DSML｜function_calls>\n<｜DSML｜invoke name="search">\n<｜DSML｜parameter name="q" string="true">v\n', ], False, [False, True], ), id="deepseek_v3_2-required", ), pytest.param( "deepseek_v4", ( {"tools": _tools_deepseek_v4_pair, "tool_choice": "required"}, [ "", '\n\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name="search">\n<｜DSML｜parameter name="q" string="true">v\n', ], False, [False, True], ), id="deepseek_v4-required", ), pytest.param( "deepseek_v3_2", ( { "tools": _tools_deepseek_v3_2_pair, "tool_choice": "forced", "forced_function_name": "search", }, [ '\n\n<｜DSML｜function_calls>\n<｜DSML｜invoke name="search">\n<｜DSML｜parameter name="q" string="true">v\n', '\n\n<｜DSML｜function_calls>\n<｜DSML｜invoke name="alt">\n<｜DSML｜parameter name="q" string="true">v\n', ], False, [True, False], ), id="deepseek_v3_2-forced", ), pytest.param( "deepseek_v4", ( { "tools": _tools_deepseek_v4_pair, "tool_choice": "forced", "forced_function_name": "search", }, [ '\n\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name="search">\n<｜DSML｜parameter name="q" string="true">v\n', '\n\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name="alt">\n<｜DSML｜parameter name="q" string="true">v\n', ], False, [True, False], ), id="deepseek_v4-forced", ), pytest.param( "minimax", ( {"tools": _tools_minimax_pair, "tool_choice": "required"}, [ "", '\n\nv\n\n', ], False, [False, False], ), id="minimax-required", ), pytest.param( "minimax", ( { "tools": _tools_minimax_pair, "tool_choice": "forced", "forced_function_name": "search", }, [ '\n\nv\n\n', '\n\nv\n\n', ], False, [False, False], ), id="minimax-forced", ), pytest.param( "qwen_3_coder", ( {"tools": _tools_qwen_3_coder_pair, "tool_choice": "required"}, [ "", "\n\nv\n\n", ], False, [False, True], ), id="qwen_coder-required", ), pytest.param( "qwen_3_coder", ( { "tools": _tools_qwen_3_coder_pair, "tool_choice": "forced", "forced_function_name": "run_sql", }, [ "\n\nv\n\n", "\n\nv\n\n", ], False, [True, False], ), id="qwen_coder-forced", ), pytest.param( "qwen_3", ( {"tools": _tools_qwen_3_pair, "tool_choice": "required"}, ["", '\n{"name": "t1", "arguments": {"q": "v"}}\n'], False, [False, True], ), id="qwen_3-required", ), pytest.param( "qwen_3", ( {"tools": _tools_qwen_3_pair, "tool_choice": "forced", "forced_function_name": "t1"}, [ '\n{"name": "t1", "arguments": {"q": "v"}}\n', '\n{"name": "t2", "arguments": {"q": "v"}}\n', ], False, [True, False], ), id="qwen_3-forced", ), pytest.param( "harmony", ( {"tools": _tools_harmony_pair, "tool_choice": "required"}, [ "plain text without channels", '<|channel|>commentary to=functions.comment_tool<|constrain|>json<|message|>{"q": "v"}<|call|>', ], False, [False, True], ), id="harmony-required", ), pytest.param( "harmony", ( { "tools": _tools_harmony_pair, "tool_choice": "forced", "forced_function_name": "comment_tool", }, [ '<|channel|>commentary to=functions.comment_tool<|constrain|>json<|message|>{"q": "v"}<|call|>', '<|channel|>commentary to=functions.other_tool<|constrain|>json<|message|>{"q": "v"}<|call|>', ], False, [True, False], ), id="harmony-forced", ), pytest.param( "glm_4_7", ( {"tools": _tools_glm_4_7_pair, "tool_choice": "required"}, ["", "searchqv"], False, [False, True], ), id="glm_4_7-required", ), pytest.param( "glm_4_7", ( { "tools": _tools_glm_4_7_pair, "tool_choice": "forced", "forced_function_name": "search", }, [ "searchqv", "altqv", ], False, [True, False], ), id="glm_4_7-forced", ), ] @pytest.mark.parametrize("format_type, case", _tool_choice_instance_cases) def test_tool_choice_instances(format_type: str, case: InstanceCase): """tool_choice required/forced: instance checks; fill expected EBNF in cases when ready.""" run_instance_case(format_type, case) _TOOLS: List[Dict[str, Any]] = [ {"function": {"name": "get_time", "parameters": {"type": "object", "properties": {}}}} ] @pytest.mark.parametrize( "format_type, kwargs", [ ("llama", {"tools": _TOOLS}), ("kimi", {"tools": _TOOLS}), ("deepseek_r1", {"tools": _TOOLS}), ("qwen_3_coder", {"tools": _TOOLS}), ("qwen_3", {"tools": _TOOLS}), ("deepseek_v3_2", {"tools": _TOOLS}), ("deepseek_v4", {"tools": _TOOLS}), ("minimax", {"tools": _TOOLS}), ( "harmony", { "tools": [ *_TOOLS, { "type": "builtin_get_time", "name": "builtin_get_time", "parameters": {"type": "object", "properties": {}}, }, ] }, ), ], ) def test_no_parameter_tools_build_grammar(format_type: str, kwargs: Dict[str, Any]): """Smoke test: each built-in format can generate StructuralTag and build Grammar.""" structural_tag = get_model_structural_tag(format_type, **kwargs) grammar = xgr.Grammar.from_structural_tag(structural_tag) assert grammar is not None # ---------- Regression: deepseek_v3_2 / deepseek_v4 parallel invoke separator ---------- # # The DeepSeek-V3.2 and DeepSeek-V4 chat templates render multiple tool calls # inside a single <｜DSML｜function_calls>... (or # <｜DSML｜tool_calls>...) wrapper, joined by a single "\n" between # and the next <｜DSML｜invoke>. Prior to this regression # guard, the built-in structural tag forced a double "\n" between consecutive # invokes (INVOKE_END's trailing "\n" plus a "\n" separator on # TagsWithSeparatorFormat), preventing the model from emitting the # in-distribution single-newline join under constrained decoding. _DSML_TOOLS_PAIR = [ { "function": { "name": "search", "parameters": {"type": "object", "properties": {"q": {"type": "string"}}}, } }, { "function": { "name": "alt", "parameters": {"type": "object", "properties": {"q": {"type": "string"}}}, } }, ] def _dsml_two_call_output(block_name: str) -> str: """Render exactly what the official chat template emits for 2 parallel calls.""" invoke_a = ( '<｜DSML｜invoke name="search">\n' '<｜DSML｜parameter name="q" string="true">v' "" ) invoke_b = ( '<｜DSML｜invoke name="alt">\n' '<｜DSML｜parameter name="q" string="true">v' "" ) tool_calls = "\n".join([invoke_a, invoke_b]) return f"<｜DSML｜{block_name}>\n{tool_calls}\n" @pytest.mark.parametrize( "model,block_name,tool_choice,prefix", [ ("deepseek_v3_2", "function_calls", "auto", ""), ("deepseek_v3_2", "function_calls", "required", "\n\n"), ("deepseek_v4", "tool_calls", "auto", ""), ("deepseek_v4", "tool_calls", "required", "\n\n"), ], ids=[ "deepseek_v3_2-auto-parallel", "deepseek_v3_2-required-parallel", "deepseek_v4-auto-parallel", "deepseek_v4-required-parallel", ], ) def test_deepseek_dsml_parallel_invokes_single_newline_accepted( model: str, block_name: str, tool_choice: str, prefix: str ): """Regression: grammar must accept the chat-template's single-\\n invoke join.""" structural_tag = get_model_structural_tag( model, tools=_DSML_TOOLS_PAIR, tool_choice=tool_choice, reasoning=False ) grammar = xgr.Grammar.from_structural_tag(structural_tag) chat_template_output = prefix + _dsml_two_call_output(block_name) assert _is_grammar_accept_string(grammar, chat_template_output), ( f"Grammar rejected chat-template output for {model}/{tool_choice}:\n" f"{chat_template_output!r}" ) @pytest.mark.parametrize( "model,block_name,tool_choice,prefix", [ ("deepseek_v3_2", "function_calls", "auto", ""), ("deepseek_v3_2", "function_calls", "required", "\n\n"), ("deepseek_v4", "tool_calls", "auto", ""), ("deepseek_v4", "tool_calls", "required", "\n\n"), ], ids=[ "deepseek_v3_2-auto-parallel", "deepseek_v3_2-required-parallel", "deepseek_v4-auto-parallel", "deepseek_v4-required-parallel", ], ) def test_deepseek_dsml_parallel_invokes_double_newline_rejected( model: str, block_name: str, tool_choice: str, prefix: str ): """Regression: grammar must NOT accept the out-of-distribution double-\\n join.""" structural_tag = get_model_structural_tag( model, tools=_DSML_TOOLS_PAIR, tool_choice=tool_choice, reasoning=False ) grammar = xgr.Grammar.from_structural_tag(structural_tag) chat_template_output = prefix + _dsml_two_call_output(block_name) double_newline_output = chat_template_output.replace( "\n<｜DSML｜invoke", "\n\n<｜DSML｜invoke", 1 ) assert not _is_grammar_accept_string(grammar, double_newline_output), ( f"Grammar wrongly accepted double-newline join for {model}/{tool_choice}:\n" f"{double_newline_output!r}" ) # ---------- Test: any_order propagation ---------- def _collect_any_order_flags(structural_tag: StructuralTag) -> List[bool]: """Collect the ``any_order`` flag of every nested JSONSchemaFormat node.""" return [ format_obj.any_order for format_obj in _walk_structural_format(structural_tag.format) if isinstance(format_obj, JSONSchemaFormat) ] _ANY_ORDER_MODELS = [ "llama", "kimi", "qwen_3", "qwen_3_5", "deepseek_r1", "deepseek_v3_1", "deepseek_v3_2", "deepseek_v4", "minimax", "glm_4_7", "harmony", ] @pytest.mark.parametrize("model", _ANY_ORDER_MODELS) @pytest.mark.parametrize("tool_choice", ["auto", "required", "forced"]) @pytest.mark.parametrize("any_order", [True, False]) def test_any_order_applies_to_every_json_schema(model: str, tool_choice: str, any_order: bool): """``any_order`` is applied to every (possibly nested) JSONSchemaFormat, for all models.""" tools = make_tools(["fn"]) choice = ( {"type": "function", "function": {"name": "fn"}} if tool_choice == "forced" else tool_choice ) structural_tag = get_model_structural_tag( model, tools=tools, tool_choice=choice, any_order=any_order ) flags = _collect_any_order_flags(structural_tag) assert flags, f"no JSONSchemaFormat found for {model}/{tool_choice}" assert all(flag is any_order for flag in flags) def test_any_order_default_is_false(): """Omitting ``any_order`` keeps the JSONSchemaFormat default (ordered).""" structural_tag = get_model_structural_tag("qwen_3", tools=make_tools(["fn"])) assert _collect_any_order_flags(structural_tag) == [False] def test_any_order_reordered_arguments_accepted_only_when_enabled(): """End-to-end: reordered tool-call arguments are accepted only when any_order=True.""" tools = [ { "function": { "name": "fn", "parameters": { "type": "object", "properties": {"a": {"type": "integer"}, "b": {"type": "string"}}, "required": ["a", "b"], "additionalProperties": False, }, } } ] forced = {"type": "function", "function": {"name": "fn"}} ordered = '\n{"name": "fn", "arguments": {"a": 1, "b": "x"}}\n' reordered = '\n{"name": "fn", "arguments": {"b": "x", "a": 1}}\n' st_ordered = get_model_structural_tag( "qwen_3", tools=tools, tool_choice=forced, reasoning=False ) check_stag_with_instance(st_ordered, ordered, True) check_stag_with_instance(st_ordered, reordered, False) st_any_order = get_model_structural_tag( "qwen_3", tools=tools, tool_choice=forced, reasoning=False, any_order=True ) check_stag_with_instance(st_any_order, ordered, True) check_stag_with_instance(st_any_order, reordered, True) xgrammar-0.2.3/tests/python/test_builtin_structural_tag_alignment.py000066400000000000000000000333731521764210300262670ustar00rootroot00000000000000"""Validate builtin structural tags against official chat templates. Uses tokenizer.apply_chat_template (or encoding scripts for DeepSeek V3.2/V4) to render model outputs, then checks that xgrammar structural tag grammars accept them. Requires encoding_dsv32.py and encoding_dsv4.py in the same directory. """ import json import os import sys from functools import lru_cache import pytest sys.path.insert(0, os.path.dirname(__file__)) from xgrammar import Grammar from xgrammar.builtin_structural_tag import get_model_structural_tag from xgrammar.testing import _is_grammar_accept_string TOOL_A = { "type": "function", "function": { "name": "get_weather", "description": "Get the weather for a location.", "parameters": { "type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"], }, }, } TOOL_B = { "type": "function", "function": { "name": "get_time", "description": "Get the current time in a timezone.", "parameters": { "type": "object", "properties": {"timezone": {"type": "string"}}, "required": ["timezone"], }, }, } USER_MSG = {"role": "user", "content": "What is the weather in Beijing?"} REASONING_CONTENT = "Let me think about this step by step." TOOL_SCENARIOS = [ (0, "auto"), (1, "auto"), (1, "forced"), (1, "required"), (2, "auto"), (2, "required"), ] # Scenarios where we render multiple parallel tool calls in the assistant # message. Used in addition to TOOL_SCENARIOS to exercise the join/separator # between consecutive invoke blocks (regression for the DeepSeek-V3.2 # double-newline grammar mismatch). PARALLEL_TOOL_SCENARIOS = [(2, "auto", 2), (2, "required", 2)] # (stag_key, model_id, reasoning, template_kwargs) # Excluded: # - Llama-4: pythonic tool call format, needs separate structural tag # - gemma_4: tool calls use <|"|> quoting, not JSON # - deepseek_r1 thinking=True: template drops in history rendering, # prompt diff extraction doesn't work # - Kimi-K2-Thinking thinking=False: model always outputs , # but grammar excludes these tokens in non-reasoning mode MODEL_CONFIGS = [ ("llama", "meta-llama/Llama-3.1-8B-Instruct", False, {}), ("kimi", "moonshotai/Kimi-K2-Thinking", True, {"thinking": True}), ("kimi", "moonshotai/Kimi-K2-Instruct", False, {}), ("deepseek_r1", "deepseek-ai/DeepSeek-R1", True, {}), ("deepseek_v3_1", "deepseek-ai/DeepSeek-V3.1", True, {"thinking": True}), ("deepseek_v3_1", "deepseek-ai/DeepSeek-V3.1", False, {"thinking": False}), ("qwen_3_coder", "Qwen/Qwen3-Coder-30B-A3B-Instruct", False, {}), ("qwen_3_coder", "Qwen/Qwen3-Coder-Next", False, {}), ("qwen_3_5", "Qwen/Qwen3.5-35B-A3B", True, {"enable_thinking": True}), ("qwen_3_5", "Qwen/Qwen3.5-35B-A3B", False, {"enable_thinking": False}), ("qwen_3_5", "Qwen/Qwen3.6-35B-A3B", True, {"enable_thinking": True}), ("qwen_3_5", "Qwen/Qwen3.6-35B-A3B", False, {"enable_thinking": False}), ("qwen_3", "Qwen/Qwen3-4B-Thinking-2507", True, {}), ("qwen_3", "Qwen/Qwen3-4B-Instruct-2507", False, {}), ("qwen_3", "Qwen/Qwen3-Next-80B-A3B-Thinking", True, {}), ("qwen_3", "Qwen/Qwen3-Next-80B-A3B-Instruct", False, {}), ("harmony", "openai/gpt-oss-20b", True, {}), ("harmony", "openai/gpt-oss-20b", False, {}), ("deepseek_v3_2", "ENCODER:dsv32", True, {"thinking_mode": "thinking"}), ("deepseek_v3_2", "ENCODER:dsv32", False, {"thinking_mode": "chat"}), ("minimax", "MiniMaxAI/MiniMax-M2.5", True, {}), ("glm_4_7", "zai-org/GLM-4.7-Flash", True, {"enable_thinking": True}), ("glm_4_7", "zai-org/GLM-4.7-Flash", False, {"enable_thinking": False}), ("deepseek_v4", "ENCODER:dsv4", True, {"thinking_mode": "thinking"}), ("deepseek_v4", "ENCODER:dsv4", False, {"thinking_mode": "chat"}), ] # DeepSeek V3.2 encoder rejects empty reasoning + no tool calls. SKIP_EMPTY_REASONING = {"ENCODER:dsv32", "MiniMaxAI/MiniMax-M2.5"} # Models where tool call format in template doesn't match structural tag. SKIP_TOOLS = set() # Models whose chat template rejects rendering more than one tool call at once # (e.g. Llama-3.1 raises "This model only supports single tool-calls at once!"). # Parallel tool-call scenarios are skipped for these. SKIP_PARALLEL_TOOLS = {"meta-llama/Llama-3.1-8B-Instruct"} # Models whose template strips or skips for empty reasoning_content, # requiring "strip base + prepend reasoning_content" to reconstruct model output. # Includes: R1/V3.1 (content.split('')), GLM/MiniMax/Qwen3.5 (falsy branch). STRIP_THINK_MODELS = { "deepseek-ai/DeepSeek-V3.1", "deepseek-ai/DeepSeek-R1", "zai-org/GLM-4.7-Flash", "MiniMaxAI/MiniMax-M2.5", "Qwen/Qwen3.5-35B-A3B", } EOS_SUFFIXES = { "llama": ["<|eot_id|>"], "kimi": ["<|im_end|>"], "deepseek_r1": ["<｜end▁of▁sentence｜>"], "deepseek_v3_1": ["<｜end▁of▁sentence｜>"], "qwen_3": ["<|im_end|>"], "qwen_3_5": ["<|im_end|>"], "qwen_3_coder": ["<|im_end|>"], "harmony": None, "minimax": ["[e~["], "glm_4_7": [], "deepseek_v3_2": ["<｜end▁of▁sentence｜>"], "deepseek_v4": ["<｜end▁of▁sentence｜>"], } @lru_cache(maxsize=None) def load_tokenizer(model_id): from transformers import AutoTokenizer return AutoTokenizer.from_pretrained(model_id, trust_remote_code=True) def make_tools(num_tools): if num_tools == 0: return None if num_tools == 1: return [TOOL_A] return [TOOL_A, TOOL_B] def make_tool_choice(choice_str, tools): if choice_str == "forced": return {"type": "function", "function": {"name": tools[0]["function"]["name"]}} return choice_str def make_assistant_msg(stag_key, reasoning_content, num_tool_calls): msg = {"role": "assistant"} if reasoning_content is not None: msg["reasoning_content"] = reasoning_content if num_tool_calls == 0: msg["content"] = "The answer is 42." return msg msg["content"] = "" tool_specs = [("get_weather", {"location": "Beijing"}), ("get_time", {"timezone": "UTC"})] calls = [] for i in range(num_tool_calls): name, args = tool_specs[i] tc = {"type": "function", "function": {"name": name}} if stag_key in ("deepseek_r1", "deepseek_v3_1"): tc["function"]["arguments"] = json.dumps(args) else: tc["function"]["arguments"] = args if stag_key == "kimi": tc["id"] = f"functions.{name}:{i}" else: tc["id"] = f"call_{i}" calls.append(tc) msg["tool_calls"] = calls return msg def strip_eos(output, stag_key, tokenizer=None): eos_list = EOS_SUFFIXES.get(stag_key) if eos_list is None: return output if not eos_list and tokenizer: eos_list = [tokenizer.eos_token] if tokenizer.eos_token else [] for eos in eos_list: if output.endswith("\n" + eos + "\n"): output = output[: -(len(eos) + 2)] break if output.endswith(eos + "\n"): output = output[: -(len(eos) + 1)] break if output.endswith("\n" + eos): output = output[: -(len(eos) + 1)] break if output.endswith(eos): output = output[: -len(eos)] break return output def extract_output_tokenizer(model_id, stag_key, assistant_msg, tools, template_kwargs): tokenizer = load_tokenizer(model_id) kwargs = dict(tokenize=False, **template_kwargs) if tools: kwargs["tools"] = tools prompt = tokenizer.apply_chat_template([USER_MSG], add_generation_prompt=True, **kwargs) full = tokenizer.apply_chat_template( [USER_MSG, assistant_msg], add_generation_prompt=False, **kwargs ) if model_id in STRIP_THINK_MODELS and assistant_msg.get("reasoning_content") is not None: if not full.startswith(prompt): base = prompt.removesuffix("\n").removesuffix("") assert full.startswith(base), ( f"Base mismatch.\nbase[-200:]={repr(base[-200:])}\n" f"full[:len(base)+200]={repr(full[: len(base) + 200])}" ) raw = full[len(base) :] raw = strip_eos(raw, stag_key, tokenizer) reasoning = assistant_msg["reasoning_content"] if not raw.startswith(""): raw = "" + raw return reasoning + raw assert full.startswith(prompt), ( f"Full does not start with prompt.\nprompt[-200:]={repr(prompt[-200:])}\n" f"full[:len(prompt)+200]={repr(full[: len(prompt) + 200])}" ) output = full[len(prompt) :] return strip_eos(output, stag_key, tokenizer) def extract_output_encoder(encoder_name, stag_key, assistant_msg, tools, template_kwargs): if encoder_name == "dsv32": from encoding_dsv32 import encode_messages, eos_token else: from encoding_dsv4 import encode_messages, eos_token thinking_mode = template_kwargs["thinking_mode"] system_msg = {"role": "system", "content": "You are a helpful assistant."} if tools: system_msg["tools"] = [{"type": "function", "function": t["function"]} for t in tools] ds_assistant = dict(assistant_msg) if "tool_calls" in ds_assistant: ds_calls = [] for tc in ds_assistant["tool_calls"]: ds_tc = dict(tc) func = dict(tc["function"]) if not isinstance(func["arguments"], str): func["arguments"] = json.dumps(func["arguments"]) ds_tc["function"] = func ds_calls.append(ds_tc) ds_assistant["tool_calls"] = ds_calls prompt = encode_messages([system_msg, USER_MSG], thinking_mode=thinking_mode) full = encode_messages([system_msg, USER_MSG, ds_assistant], thinking_mode=thinking_mode) assert full.startswith(prompt) output = full[len(prompt) :] if output.endswith(eos_token): output = output[: -len(eos_token)] return output def extract_model_output(stag_key, model_id, assistant_msg, tools, template_kwargs): if model_id.startswith("ENCODER:"): encoder_name = model_id.split(":")[1] return extract_output_encoder(encoder_name, stag_key, assistant_msg, tools, template_kwargs) return extract_output_tokenizer(model_id, stag_key, assistant_msg, tools, template_kwargs) def validate_output(stag_key, tools, tool_choice, reasoning, model_output): structural_tag = get_model_structural_tag( stag_key, tools=tools or [], tool_choice=tool_choice, reasoning=reasoning ) grammar = Grammar.from_structural_tag(structural_tag) accepted = _is_grammar_accept_string(grammar, model_output) assert accepted, f"Grammar rejected output:\n{repr(model_output[:500])}" def generate_test_cases(): cases = [] for stag_key, model_id, reasoning, template_kwargs in MODEL_CONFIGS: scenarios = [(n, c, 1 if n > 0 else 0) for n, c in TOOL_SCENARIOS] scenarios.extend(PARALLEL_TOOL_SCENARIOS) for num_tools, tool_choice_str, num_tool_calls in scenarios: if num_tools > 0 and model_id in SKIP_TOOLS: continue if num_tool_calls > 1 and model_id in SKIP_PARALLEL_TOOLS: continue if reasoning: cases.append( ( stag_key, model_id, True, REASONING_CONTENT, num_tools, tool_choice_str, template_kwargs, num_tool_calls, ) ) if model_id not in SKIP_EMPTY_REASONING: cases.append( ( stag_key, model_id, True, "", num_tools, tool_choice_str, template_kwargs, num_tool_calls, ) ) else: cases.append( ( stag_key, model_id, False, None, num_tools, tool_choice_str, template_kwargs, num_tool_calls, ) ) return cases def case_id(case): ( stag_key, model_id, reasoning, reasoning_content, num_tools, tool_choice_str, _, num_tool_calls, ) = case model_short = model_id.split("/")[-1] if "/" in model_id else model_id.replace("ENCODER:", "") if not reasoning: r_tag = "off" elif reasoning_content: r_tag = "on" else: r_tag = "empty" return f"{stag_key}-{model_short}-r{r_tag}-{num_tools}t-{tool_choice_str}-{num_tool_calls}calls" TEST_CASES = generate_test_cases() @pytest.mark.hf_token_required @pytest.mark.parametrize("case", TEST_CASES, ids=[case_id(c) for c in TEST_CASES]) def test_reasoning_stag(case): ( stag_key, model_id, reasoning, reasoning_content, num_tools, tool_choice_str, template_kwargs, num_tool_calls, ) = case tools = make_tools(num_tools) tool_choice = make_tool_choice(tool_choice_str, tools or []) assistant_msg = make_assistant_msg(stag_key, reasoning_content, num_tool_calls) model_output = extract_model_output(stag_key, model_id, assistant_msg, tools, template_kwargs) validate_output(stag_key, tools, tool_choice, reasoning, model_output) if __name__ == "__main__": pytest.main(["-v", __file__]) xgrammar-0.2.3/tests/python/test_function_calling_converter.py000066400000000000000000002366331521764210300250510ustar00rootroot00000000000000import sys import pytest from xgrammar import Grammar from xgrammar.testing import ( _deepseek_xml_tool_calling_to_ebnf, _glm_xml_tool_calling_to_ebnf, _is_grammar_accept_string, _minimax_xml_tool_calling_to_ebnf, _qwen_xml_tool_calling_to_ebnf, ) def check_grammar_with_expected_grammar(grammar: Grammar, expected_grammar: str): assert ( str(grammar).rstrip() == expected_grammar.rstrip() ), f"Expected grammar:\n{expected_grammar}\nActual grammar:\n{str(grammar)}" def check_grammar_with_instance(grammar: Grammar, instance: str, accepted: bool): assert _is_grammar_accept_string(grammar, instance) == accepted def _check_qwen_grammar(schema: dict, expected_grammar: str, instance: str, accepted: bool): ebnf_grammar = _qwen_xml_tool_calling_to_ebnf(schema) check_grammar_with_expected_grammar(ebnf_grammar, expected_grammar) check_grammar_with_instance(ebnf_grammar, instance, accepted) def _check_minimax_grammar(schema: dict, expected_grammar: str, instance: str, accepted: bool): ebnf_grammar = _minimax_xml_tool_calling_to_ebnf(schema) check_grammar_with_expected_grammar(ebnf_grammar, expected_grammar) check_grammar_with_instance(ebnf_grammar, instance, accepted) def _check_deepseek_grammar(schema: dict, expected_grammar: str, instance: str, accepted: bool): ebnf_grammar = _deepseek_xml_tool_calling_to_ebnf(schema) check_grammar_with_expected_grammar(ebnf_grammar, expected_grammar) check_grammar_with_instance(ebnf_grammar, instance, accepted) def _check_glm_grammar(schema: dict, instance: str, accepted: bool): ebnf_grammar = _glm_xml_tool_calling_to_ebnf(schema) check_grammar_with_instance(ebnf_grammar, instance, accepted) test_string_schema_input_str_accepted = ( ("Bob\t100\n", True), ("Bob\t\n\t100\n", True), ("Bob100", True), ( """

Hello

100""", True, ), ) @pytest.mark.parametrize("input_str, accepted", test_string_schema_input_str_accepted) def test_string_schema(input_str: str, accepted: bool): expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_1 ::= ("0" | "-"? [1-9] [0-9]*) root_part_0 ::= [ \n\t]* "" [ \n\t]* root_prop_1 [ \n\t]* "" "" root ::= [ \n\t]* (("" [ \n\t]* xml_string [ \n\t]* "" root_part_0)) [ \n\t]* """ schema = { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], } _check_qwen_grammar(schema, expected_grammar, input_str, accepted) test_additional_properties_schema_input_str_accepted = ( ( "Bob\t100\nNew York", True, ), ( "Bob100A", False, ), ) @pytest.mark.parametrize( "input_str, accepted", test_additional_properties_schema_input_str_accepted ) def test_additional_properties_schema(input_str: str, accepted: bool): expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_1 ::= ("0" | "-"? [1-9] [0-9]*) root_addl ::= xml_string | basic_array | basic_object root_part_1 ::= ([ \n\t]* "" [ \n\t]* root_addl [ \n\t]* "")* root_part_0 ::= [ \n\t]* "" [ \n\t]* root_prop_1 [ \n\t]* "" root_part_1 root ::= [ \n\t]* (("" [ \n\t]* xml_string [ \n\t]* "" root_part_0)) [ \n\t]* """ schema = { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], "additionalProperties": True, } _check_qwen_grammar(schema, expected_grammar, input_str, accepted) test_not_required_properties_schema_input_str_accepted = ( ("Bob\t100\n", True), ("Bob", True), ("100", True), ("", True), ("It's a string.", True), ) @pytest.mark.parametrize( "input_str, accepted", test_not_required_properties_schema_input_str_accepted ) def test_not_required_properties_schema(input_str: str, accepted: bool): expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_1 ::= ("0" | "-"? [1-9] [0-9]*) root_addl ::= xml_string | basic_array | basic_object root_part_1 ::= ([ \n\t]* "" [ \n\t]* root_addl [ \n\t]* "")* root_part_0 ::= root_part_1 | [ \n\t]* "" [ \n\t]* root_prop_1 [ \n\t]* "" root_part_1 root ::= ( [ \n\t]* (("" [ \n\t]* xml_string [ \n\t]* "" root_part_0) | ("" [ \n\t]* root_prop_1 [ \n\t]* "" root_part_1) | "" [ \n\t]* root_addl [ \n\t]* "" root_part_1) [ \n\t]*) | [ \n\t]* """ schema = { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "additionalProperties": True, } _check_qwen_grammar(schema, expected_grammar, input_str, accepted) test_part_required_properties_schema_input_str_accepted = ( ("Bob\t100\n", True), ("Bob", True), ("100", False), ( "Bob\t100\nIt's a string.", True, ), ("BobIt's a string.", True), ("It's a string.", False), ) @pytest.mark.parametrize( "input_str, accepted", test_part_required_properties_schema_input_str_accepted ) def test_part_required_properties_schema(input_str: str, accepted: bool): expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_1 ::= ("0" | "-"? [1-9] [0-9]*) root_addl ::= xml_string | basic_array | basic_object root_part_1 ::= ([ \n\t]* "" [ \n\t]* root_addl [ \n\t]* "")* root_part_0 ::= root_part_1 | [ \n\t]* "" [ \n\t]* root_prop_1 [ \n\t]* "" root_part_1 root ::= [ \n\t]* (("" [ \n\t]* xml_string [ \n\t]* "" root_part_0)) [ \n\t]* """ schema = { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name"], "additionalProperties": True, } _check_qwen_grammar(schema, expected_grammar, input_str, accepted) test_inner_object_schema_input_str_accepted = ( ('{"street": "Main St", "city": "New York"}', True), ('{"street": "Main St", "city": "No more xml escape&<>"}', True), ('{"street": Main St, "city": New York}', False), ( "Main StNew York", False, ), ('{"street": "Main St"}', False), ('{"city": "New York"}', False), ) @pytest.mark.parametrize("input_str, accepted", test_inner_object_schema_input_str_accepted) def test_inner_object_schema(input_str: str, accepted: bool): expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_0_part_0 ::= [ \n\t]* "," [ \n\t]* "\"city\"" [ \n\t]* ":" [ \n\t]* basic_string "" root_prop_0 ::= "{" [ \n\t]* (("\"street\"" [ \n\t]* ":" [ \n\t]* basic_string root_prop_0_part_0)) [ \n\t]* "}" root ::= [ \n\t]* (("" [ \n\t]* root_prop_0 [ \n\t]* "" "")) [ \n\t]* """ schema = { "type": "object", "properties": { "address": { "type": "object", "properties": {"street": {"type": "string"}, "city": {"type": "string"}}, "required": ["street", "city"], } }, "required": ["address"], } _check_qwen_grammar(schema, expected_grammar, input_str, accepted) test_numbers_schema_input_str_accepted = ( ("25", False), ("Bob25", True), ( "Bob123456true", True, ), ( "John11false", False, ), ) @pytest.mark.parametrize("input_str, accepted", test_numbers_schema_input_str_accepted) def test_numbers_schema(input_str: str, accepted: bool): expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_1 ::= ("0" | "-"? [1-9] [0-9]*) root_prop_2 ::= ("0" | "-"? [1-9] [0-9]*) root_prop_3 ::= "true" | "false" root_part_2_1 ::= [ \n\t]* "" [ \n\t]* root_prop_3 [ \n\t]* "" "" root_part_2_2 ::= "" | [ \n\t]* "" [ \n\t]* root_prop_3 [ \n\t]* "" "" root_part_2_3 ::= "" root_part_1_1 ::= root_part_2_1 | [ \n\t]* "" [ \n\t]* root_prop_2 [ \n\t]* "" root_part_2_2 root_part_1_2 ::= root_part_2_2 | [ \n\t]* "" [ \n\t]* root_prop_2 [ \n\t]* "" root_part_2_3 root_part_0_1 ::= root_part_1_1 | [ \n\t]* "" [ \n\t]* root_prop_1 [ \n\t]* "" root_part_1_2 root ::= [ \n\t]* (("" [ \n\t]* xml_string [ \n\t]* "" root_part_0_1) | ("" [ \n\t]* root_prop_1 [ \n\t]* "" root_part_1_1) | ("" [ \n\t]* root_prop_2 [ \n\t]* "" root_part_2_1)) [ \n\t]* """ schema = { "type": "object", "properties": { "name": {"type": "string"}, "age": {"type": "integer"}, "ID": {"type": "integer"}, "is_student": {"type": "boolean"}, }, "maxProperties": 3, "minProperties": 2, } _check_qwen_grammar(schema, expected_grammar, input_str, accepted) test_string_format_length_schema_input_str_accepted = { ( 'ABC{"phone": "12345", "email": "test@test.com"}', True, ), ( 'X{"phone": "67890", "email": "a@b.com"}', True, ), ( '{"phone": "12345", "email": "test@test.com"}', False, ), ( 'ABC{"phone": "1234", "email": "test@test.com"}', False, ), ( 'ABC{"phone": "12345", "email": "not-an-email"}', False, ), ( 'ABC{"phone": "12345"}', False, ), ( 'ABC{"email": "test@test.com"}', False, ), ("ABC", False), ('{"phone": "12345", "email": "test@test.com"}', False), } @pytest.mark.parametrize("input_str, accepted", test_string_format_length_schema_input_str_accepted) def test_string_format_length_schema(input_str: str, accepted: bool): expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_0 ::= [^]{1,} root_prop_1_prop_0 ::= "\"" [0-9]{5} "\"" root_prop_1_prop_1 ::= "\"" ( ( [a-zA-Z0-9_!#$%&'*+/=?^`{|}~-]+ ( "." [a-zA-Z0-9_!#$%&'*+/=?^`{|}~-]+ )* ) | "\\" "\"" ( "\\" [ -~] | [ !#-[\]-~] )* "\\" "\"" ) "@" ( [A-Za-z0-9] ( [\-A-Za-z0-9]* [A-Za-z0-9] )? ) ( ( "." [A-Za-z0-9] [\-A-Za-z0-9]* [A-Za-z0-9] )* ) "\"" root_prop_1_part_0 ::= [ \n\t]* "," [ \n\t]* "\"email\"" [ \n\t]* ":" [ \n\t]* root_prop_1_prop_1 "" root_prop_1 ::= "{" [ \n\t]* (("\"phone\"" [ \n\t]* ":" [ \n\t]* root_prop_1_prop_0 root_prop_1_part_0)) [ \n\t]* "}" root_part_0 ::= [ \n\t]* "" [ \n\t]* root_prop_1 [ \n\t]* "" "" root ::= [ \n\t]* (("" [ \n\t]* root_prop_0 [ \n\t]* "" root_part_0)) [ \n\t]* """ schema = { "type": "object", "properties": { "name": {"type": "string", "minLength": 1}, "contact_info": { "type": "object", "properties": { "phone": {"type": "string", "pattern": "[0-9]{5}$"}, "email": {"type": "string", "format": "email"}, }, "required": ["phone", "email"], }, }, "required": ["name", "contact_info"], } _check_qwen_grammar(schema, expected_grammar, input_str, accepted) test_array_schema_input_str_accepted = ( ('["foo", "bar"]', True), ('["foo", "bar", "baz"]', True), ("[]", True), ("[foo, bar, baz, qux, quux, corge]", False), ) @pytest.mark.parametrize("input_str, accepted", test_array_schema_input_str_accepted) def test_array_schema(input_str: str, accepted: bool): expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_0 ::= (("[" [ \n\t]* basic_string ([ \n\t]* "," [ \n\t]* basic_string)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) root ::= [ \n\t]* (("" [ \n\t]* root_prop_0 [ \n\t]* "" "")) [ \n\t]* """ schema = { "type": "object", "properties": {"array": {"type": "array", "items": {"type": "string"}}}, "required": ["array"], } _check_qwen_grammar(schema, expected_grammar, input_str, accepted) # ---------- MiniMax XML tool calling (_minimax_xml_tool_calling_to_ebnf) ---------- # Format: value (not ) minimax_test_string_schema_input_str_accepted = ( ('Bob\t100\n', True), ('Bob\t\n\t100\n', True), ('Bob100', True), ( """

Hello

100""", True, ), ) @pytest.mark.parametrize("input_str, accepted", minimax_test_string_schema_input_str_accepted) def test_minimax_string_schema(input_str: str, accepted: bool): expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_1 ::= ("0" | "-"? [1-9] [0-9]*) root_part_0 ::= [ \n\t]* "" [ \n\t]* root_prop_1 [ \n\t]* "" "" root ::= [ \n\t]* (("" [ \n\t]* xml_string [ \n\t]* "" root_part_0)) [ \n\t]* """ schema = { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], } _check_minimax_grammar(schema, expected_grammar, input_str, accepted) minimax_test_additional_properties_schema_input_str_accepted = ( ( 'Bob\t100\nNew York', True, ), ( 'Bob100A', False, ), ) @pytest.mark.parametrize( "input_str, accepted", minimax_test_additional_properties_schema_input_str_accepted ) def test_minimax_additional_properties_schema(input_str: str, accepted: bool): expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_1 ::= ("0" | "-"? [1-9] [0-9]*) root_addl ::= xml_string | basic_array | basic_object root_part_1 ::= ([ \n\t]* "" [ \n\t]* root_addl [ \n\t]* "")* root_part_0 ::= [ \n\t]* "" [ \n\t]* root_prop_1 [ \n\t]* "" root_part_1 root ::= [ \n\t]* (("" [ \n\t]* xml_string [ \n\t]* "" root_part_0)) [ \n\t]* """ schema = { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], "additionalProperties": True, } _check_minimax_grammar(schema, expected_grammar, input_str, accepted) minimax_test_not_required_properties_schema_input_str_accepted = ( ('Bob\t100\n', True), ('Bob', True), ('100', True), ("", True), ('It\'s a string.', True), ) @pytest.mark.parametrize( "input_str, accepted", minimax_test_not_required_properties_schema_input_str_accepted ) def test_minimax_not_required_properties_schema(input_str: str, accepted: bool): expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_1 ::= ("0" | "-"? [1-9] [0-9]*) root_addl ::= xml_string | basic_array | basic_object root_part_1 ::= ([ \n\t]* "" [ \n\t]* root_addl [ \n\t]* "")* root_part_0 ::= root_part_1 | [ \n\t]* "" [ \n\t]* root_prop_1 [ \n\t]* "" root_part_1 root ::= ( [ \n\t]* (("" [ \n\t]* xml_string [ \n\t]* "" root_part_0) | ("" [ \n\t]* root_prop_1 [ \n\t]* "" root_part_1) | "" [ \n\t]* root_addl [ \n\t]* "" root_part_1) [ \n\t]*) | [ \n\t]* """ schema = { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "additionalProperties": True, } _check_minimax_grammar(schema, expected_grammar, input_str, accepted) minimax_test_part_required_properties_schema_input_str_accepted = ( ('Bob\t100\n', True), ('Bob', True), ('100', False), ( 'Bob\t100\nIt\'s a string.', True, ), ( 'BobIt\'s a string.', True, ), ('It\'s a string.', False), ) @pytest.mark.parametrize( "input_str, accepted", minimax_test_part_required_properties_schema_input_str_accepted ) def test_minimax_part_required_properties_schema(input_str: str, accepted: bool): expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_1 ::= ("0" | "-"? [1-9] [0-9]*) root_addl ::= xml_string | basic_array | basic_object root_part_1 ::= ([ \n\t]* "" [ \n\t]* root_addl [ \n\t]* "")* root_part_0 ::= root_part_1 | [ \n\t]* "" [ \n\t]* root_prop_1 [ \n\t]* "" root_part_1 root ::= [ \n\t]* (("" [ \n\t]* xml_string [ \n\t]* "" root_part_0)) [ \n\t]* """ schema = { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name"], "additionalProperties": True, } _check_minimax_grammar(schema, expected_grammar, input_str, accepted) minimax_test_inner_object_schema_input_str_accepted = ( ('{"street": "Main St", "city": "New York"}', True), ( '{"street": "Main St", "city": "No more xml escape&<>"}', True, ), ('{"street": Main St, "city": New York}', False), ( 'Main StNew York', False, ), ('{"street": "Main St"}', False), ('{"city": "New York"}', False), ( '{"street": "Main St", "city": "New York", "additional_property": "value"}value', True, ), ( '{"street": "Main St", "city": "New York", "additional_property": value}', False, ), ) @pytest.mark.parametrize("input_str, accepted", minimax_test_inner_object_schema_input_str_accepted) def test_minimax_inner_object_schema(input_str: str, accepted: bool): expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_0_addl ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object root_prop_0_addl_key ::= ["] (("\"" | [^cs\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "c" ("\"" | [^i\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "i" ("\"" | [^t\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "t" ("\"" | [^y\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "y" ([^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub)))) | "s" ("\"" | [^t\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "t" ("\"" | [^r\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "r" ("\"" | [^e\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "e" ("\"" | [^e\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "e" ("\"" | [^t\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "t" ([^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub)))))))) (= [ \n\t]* [,}\]:]) root_prop_0_part_1 ::= ([ \n\t]* "," [ \n\t]* root_prop_0_addl_key [ \n\t]* ":" [ \n\t]* root_prop_0_addl)* root_prop_0_part_0 ::= [ \n\t]* "," [ \n\t]* "\"city\"" [ \n\t]* ":" [ \n\t]* basic_string root_prop_0_part_1 root_prop_0 ::= "{" [ \n\t]* (("\"street\"" [ \n\t]* ":" [ \n\t]* basic_string root_prop_0_part_0)) [ \n\t]* "}" root_addl ::= xml_string | basic_array | basic_object root_part_0 ::= ([ \n\t]* "" [ \n\t]* root_addl [ \n\t]* "")* root ::= [ \n\t]* (("" [ \n\t]* root_prop_0 [ \n\t]* "" root_part_0)) [ \n\t]* """ schema = { "type": "object", "properties": { "address": { "type": "object", "properties": {"street": {"type": "string"}, "city": {"type": "string"}}, "required": ["street", "city"], "additionalProperties": True, } }, "additionalProperties": True, "required": ["address"], } _check_minimax_grammar(schema, expected_grammar, input_str, accepted) minimax_test_numbers_schema_input_str_accepted = ( ('25', False), ('Bob25', True), ( 'Bob123456true', True, ), ( 'John11false', False, ), ) @pytest.mark.parametrize("input_str, accepted", minimax_test_numbers_schema_input_str_accepted) def test_minimax_numbers_schema(input_str: str, accepted: bool): expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_1 ::= ("0" | "-"? [1-9] [0-9]*) root_prop_2 ::= ("0" | "-"? [1-9] [0-9]*) root_prop_3 ::= "true" | "false" root_part_2_1 ::= [ \n\t]* "" [ \n\t]* root_prop_3 [ \n\t]* "" "" root_part_2_2 ::= "" | [ \n\t]* "" [ \n\t]* root_prop_3 [ \n\t]* "" "" root_part_2_3 ::= "" root_part_1_1 ::= root_part_2_1 | [ \n\t]* "" [ \n\t]* root_prop_2 [ \n\t]* "" root_part_2_2 root_part_1_2 ::= root_part_2_2 | [ \n\t]* "" [ \n\t]* root_prop_2 [ \n\t]* "" root_part_2_3 root_part_0_1 ::= root_part_1_1 | [ \n\t]* "" [ \n\t]* root_prop_1 [ \n\t]* "" root_part_1_2 root ::= [ \n\t]* (("" [ \n\t]* xml_string [ \n\t]* "" root_part_0_1) | ("" [ \n\t]* root_prop_1 [ \n\t]* "" root_part_1_1) | ("" [ \n\t]* root_prop_2 [ \n\t]* "" root_part_2_1)) [ \n\t]* """ schema = { "type": "object", "properties": { "name": {"type": "string"}, "age": {"type": "integer"}, "ID": {"type": "integer"}, "is_student": {"type": "boolean"}, }, "maxProperties": 3, "minProperties": 2, } _check_minimax_grammar(schema, expected_grammar, input_str, accepted) minimax_test_string_format_length_schema_input_str_accepted = ( ( 'ABC{"phone": "12345", "email": "test@test.com"}', True, ), ( 'X{"phone": "67890", "email": "a@b.com"}', True, ), ( '{"phone": "12345", "email": "test@test.com"}', False, ), ( 'ABC{"phone": "1234", "email": "test@test.com"}', False, ), ( 'ABC{"phone": "12345", "email": "not-an-email"}', False, ), ( 'ABC{"phone": "12345"}', False, ), ( 'ABC{"email": "test@test.com"}', False, ), ('ABC', False), ( '{"phone": "12345", "email": "test@test.com"}', False, ), ) @pytest.mark.parametrize( "input_str, accepted", minimax_test_string_format_length_schema_input_str_accepted ) def test_minimax_string_format_length_schema(input_str: str, accepted: bool): expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_0 ::= [^]{1,} root_prop_1_prop_0 ::= "\"" [0-9]{5} "\"" root_prop_1_prop_1 ::= "\"" ( ( [a-zA-Z0-9_!#$%&'*+/=?^`{|}~-]+ ( "." [a-zA-Z0-9_!#$%&'*+/=?^`{|}~-]+ )* ) | "\\" "\"" ( "\\" [ -~] | [ !#-[\]-~] )* "\\" "\"" ) "@" ( [A-Za-z0-9] ( [\-A-Za-z0-9]* [A-Za-z0-9] )? ) ( ( "." [A-Za-z0-9] [\-A-Za-z0-9]* [A-Za-z0-9] )* ) "\"" root_prop_1_part_0 ::= [ \n\t]* "," [ \n\t]* "\"email\"" [ \n\t]* ":" [ \n\t]* root_prop_1_prop_1 "" root_prop_1 ::= "{" [ \n\t]* (("\"phone\"" [ \n\t]* ":" [ \n\t]* root_prop_1_prop_0 root_prop_1_part_0)) [ \n\t]* "}" root_part_0 ::= [ \n\t]* "" [ \n\t]* root_prop_1 [ \n\t]* "" "" root ::= [ \n\t]* (("" [ \n\t]* root_prop_0 [ \n\t]* "" root_part_0)) [ \n\t]* """ schema = { "type": "object", "properties": { "name": {"type": "string", "minLength": 1}, "contact_info": { "type": "object", "properties": { "phone": {"type": "string", "pattern": "[0-9]{5}$"}, "email": {"type": "string", "format": "email"}, }, "required": ["phone", "email"], }, }, "required": ["name", "contact_info"], } _check_minimax_grammar(schema, expected_grammar, input_str, accepted) # Minimax: reject Qwen format and unquoted minimax_reject_wrong_parameter_format_input_str_accepted = ( ("Bob100", False), # Qwen format ( "Bob100", False, ), # unquoted key ( 'Bob100', True, ), # correct ) @pytest.mark.parametrize( "input_str, accepted", minimax_reject_wrong_parameter_format_input_str_accepted ) def test_minimax_reject_wrong_parameter_format(input_str: str, accepted: bool): """MiniMax grammar must accept but reject and .""" expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_1 ::= ("0" | "-"? [1-9] [0-9]*) root_part_0 ::= [ \n\t]* "" [ \n\t]* root_prop_1 [ \n\t]* "" "" root ::= [ \n\t]* (("" [ \n\t]* xml_string [ \n\t]* "" root_part_0)) [ \n\t]* """ schema = { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], } _check_minimax_grammar(schema, expected_grammar, input_str, accepted) # ---------- DeepSeek XML tool calling (_deepseek_xml_tool_calling_to_ebnf) ---------- # Format: <｜DSML｜parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE deepseek_test_string_schema_input_str_accepted = ( ( '<｜DSML｜parameter name="name" string="true">Bob<｜DSML｜parameter name="age" string="false">\t100\n', True, ), ( '<｜DSML｜parameter name="name" string="true">Bob\t\n<｜DSML｜parameter name="age" string="true">\t100\n', True, ), ( '<｜DSML｜parameter name="name" string="false">Bob<｜DSML｜parameter name="age" string="true">100', True, ), ( """<｜DSML｜parameter name="name" string="true">

Hello

<｜DSML｜parameter name="age" string="false">100""", True, ), ('<｜DSML｜parameter name="name" string="true">Bob', False), ('<｜DSML｜parameter name="age" string="false">100', False), ( '<｜DSML｜parameter name="name" string="true">Bob<｜DSML｜parameter name="age" string="false">100', False, ), ( '<｜DSML｜parameter name="name">Bob<｜DSML｜parameter name="age" string="false">100', False, ), ( '<｜DSML｜parameter name="name" string="true">Bob<｜DSML｜parameter name="age" string="false">100', False, ), ) @pytest.mark.parametrize("input_str, accepted", deepseek_test_string_schema_input_str_accepted) def test_deepseek_string_schema(input_str: str, accepted: bool): expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "<｜DSML｜parameter name=\"" xml_variable_name "\" string=\"" ("true" | "false") "\">" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "<｜DSML｜parameter name=\"" xml_variable_name "\" string=\"" ("true" | "false") "\">" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_1 ::= ("0" | "-"? [1-9] [0-9]*) root_part_0 ::= [ \n\t]* "<｜DSML｜parameter name=\"age\" string=\"" ("true" | "false") "\">" [ \n\t]* root_prop_1 [ \n\t]* "" "" root ::= [ \n\t]* (("<｜DSML｜parameter name=\"name\" string=\"" ("true" | "false") "\">" [ \n\t]* xml_string [ \n\t]* "" root_part_0)) [ \n\t]* """ schema = { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], } _check_deepseek_grammar(schema, expected_grammar, input_str, accepted) deepseek_pattern_empty_leading_alternative_input_str_accepted = ( ('<｜DSML｜parameter name="url" string="true">https://x.com/', True), # The "^$" branch allows an empty value. ('<｜DSML｜parameter name="url" string="true">', True), ('<｜DSML｜parameter name="url" string="true">http://x.com/', False), ) @pytest.mark.parametrize( "input_str, accepted", deepseek_pattern_empty_leading_alternative_input_str_accepted ) def test_deepseek_pattern_empty_leading_alternative(input_str: str, accepted: bool): # Regression: a pattern whose first alternative is empty ("^$|...") used to emit a bare # leading '|' (root_prop_0 ::= | ...) and crash the grammar parser on the deepseek_xml path. # It must now be emitted as root_prop_0 ::= "" | ... expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "<｜DSML｜parameter name=\"" xml_variable_name "\" string=\"" ("true" | "false") "\">" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "<｜DSML｜parameter name=\"" xml_variable_name "\" string=\"" ("true" | "false") "\">" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_0 ::= "" | "h" "t" "t" "p" "s" ":" "/" "/" "x" "." "c" "o" "m" "/" root ::= [ \n\t]* (("<｜DSML｜parameter name=\"url\" string=\"" ("true" | "false") "\">" [ \n\t]* root_prop_0 [ \n\t]* "" "")) [ \n\t]* """ schema = { "type": "object", "properties": {"url": {"type": "string", "pattern": "^$|^https://x\\.com/"}}, "required": ["url"], } _check_deepseek_grammar(schema, expected_grammar, input_str, accepted) deepseek_test_additional_properties_schema_input_str_accepted = ( ( '<｜DSML｜parameter name="name" string="true">Bob<｜DSML｜parameter name="age" string="false">\t100\n<｜DSML｜parameter name="location" string="true">New York', True, ), ( '<｜DSML｜parameter name="name" string="true">Bob<｜DSML｜parameter name="age" string="true">100<｜DSML｜parameter name="123invalid" string="false">A', False, ), ('<｜DSML｜parameter name="location" string="true">New York', False), ('<｜DSML｜parameter name="name" string="true">Bob', False), ( '<｜DSML｜parameter name="name" string="true">Bob<｜DSML｜parameter name="age" string="false">100', False, ), ) @pytest.mark.parametrize( "input_str, accepted", deepseek_test_additional_properties_schema_input_str_accepted ) def test_deepseek_additional_properties_schema(input_str: str, accepted: bool): expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "<｜DSML｜parameter name=\"" xml_variable_name "\" string=\"" ("true" | "false") "\">" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "<｜DSML｜parameter name=\"" xml_variable_name "\" string=\"" ("true" | "false") "\">" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_1 ::= ("0" | "-"? [1-9] [0-9]*) root_addl ::= xml_string | basic_array | basic_object root_part_1 ::= ([ \n\t]* "<｜DSML｜parameter name=\"" xml_variable_name "\" string=\"" ("true" | "false") "\">" [ \n\t]* root_addl [ \n\t]* "")* root_part_0 ::= [ \n\t]* "<｜DSML｜parameter name=\"age\" string=\"" ("true" | "false") "\">" [ \n\t]* root_prop_1 [ \n\t]* "" root_part_1 root ::= [ \n\t]* (("<｜DSML｜parameter name=\"name\" string=\"" ("true" | "false") "\">" [ \n\t]* xml_string [ \n\t]* "" root_part_0)) [ \n\t]* """ schema = { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], "additionalProperties": True, } _check_deepseek_grammar(schema, expected_grammar, input_str, accepted) deepseek_test_not_required_properties_schema_input_str_accepted = ( ( '<｜DSML｜parameter name="name" string="true">Bob<｜DSML｜parameter name="age" string="false">\t100\n', True, ), ('<｜DSML｜parameter name="name" string="true">Bob', True), ('<｜DSML｜parameter name="age" string="false">100', True), ("", True), ('<｜DSML｜parameter name="anything" string="true">It\'s a string.', True), ('<｜DSML｜parameter name="name" string="true">Bob', False), ('<｜DSML｜parameter name="name">Bob', False), ('<｜DSML｜parameter name="x" string="true">y', False), ) @pytest.mark.parametrize( "input_str, accepted", deepseek_test_not_required_properties_schema_input_str_accepted ) def test_deepseek_not_required_properties_schema(input_str: str, accepted: bool): expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "<｜DSML｜parameter name=\"" xml_variable_name "\" string=\"" ("true" | "false") "\">" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "<｜DSML｜parameter name=\"" xml_variable_name "\" string=\"" ("true" | "false") "\">" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_1 ::= ("0" | "-"? [1-9] [0-9]*) root_addl ::= xml_string | basic_array | basic_object root_part_1 ::= ([ \n\t]* "<｜DSML｜parameter name=\"" xml_variable_name "\" string=\"" ("true" | "false") "\">" [ \n\t]* root_addl [ \n\t]* "")* root_part_0 ::= root_part_1 | [ \n\t]* "<｜DSML｜parameter name=\"age\" string=\"" ("true" | "false") "\">" [ \n\t]* root_prop_1 [ \n\t]* "" root_part_1 root ::= ( [ \n\t]* (("<｜DSML｜parameter name=\"name\" string=\"" ("true" | "false") "\">" [ \n\t]* xml_string [ \n\t]* "" root_part_0) | ("<｜DSML｜parameter name=\"age\" string=\"" ("true" | "false") "\">" [ \n\t]* root_prop_1 [ \n\t]* "" root_part_1) | "<｜DSML｜parameter name=\"" xml_variable_name "\" string=\"" ("true" | "false") "\">" [ \n\t]* root_addl [ \n\t]* "" root_part_1) [ \n\t]*) | [ \n\t]* """ schema = { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "additionalProperties": True, } _check_deepseek_grammar(schema, expected_grammar, input_str, accepted) deepseek_test_part_required_properties_schema_input_str_accepted = ( ( '<｜DSML｜parameter name="name" string="true">Bob<｜DSML｜parameter name="age" string="false">\t100\n', True, ), ('<｜DSML｜parameter name="name" string="true">Bob', True), ('<｜DSML｜parameter name="age" string="true">100', False), ( '<｜DSML｜parameter name="name" string="true">Bob<｜DSML｜parameter name="age" string="false">\t100\n<｜DSML｜parameter name="anything" string="true">It\'s a string.', True, ), ( '<｜DSML｜parameter name="name" string="false">Bob<｜DSML｜parameter name="anything" string="true">It\'s a string.', True, ), ('<｜DSML｜parameter name="anything" string="true">It\'s a string.', False), ) @pytest.mark.parametrize( "input_str, accepted", deepseek_test_part_required_properties_schema_input_str_accepted ) def test_deepseek_part_required_properties_schema(input_str: str, accepted: bool): expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "<｜DSML｜parameter name=\"" xml_variable_name "\" string=\"" ("true" | "false") "\">" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "<｜DSML｜parameter name=\"" xml_variable_name "\" string=\"" ("true" | "false") "\">" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_1 ::= ("0" | "-"? [1-9] [0-9]*) root_addl ::= xml_string | basic_array | basic_object root_part_1 ::= ([ \n\t]* "<｜DSML｜parameter name=\"" xml_variable_name "\" string=\"" ("true" | "false") "\">" [ \n\t]* root_addl [ \n\t]* "")* root_part_0 ::= root_part_1 | [ \n\t]* "<｜DSML｜parameter name=\"age\" string=\"" ("true" | "false") "\">" [ \n\t]* root_prop_1 [ \n\t]* "" root_part_1 root ::= [ \n\t]* (("<｜DSML｜parameter name=\"name\" string=\"" ("true" | "false") "\">" [ \n\t]* xml_string [ \n\t]* "" root_part_0)) [ \n\t]* """ schema = { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name"], "additionalProperties": True, } _check_deepseek_grammar(schema, expected_grammar, input_str, accepted) deepseek_test_inner_object_schema_input_str_accepted = ( ( '<｜DSML｜parameter name="address" string="true">{"street": "Main St", "city": "New York"}', True, ), ( '<｜DSML｜parameter name="address" string="false">{"street": "Main St", "city": "No more xml escape&<>"}', True, ), ( '<｜DSML｜parameter name="address" string="true">{"street": Main St, "city": New York}', False, ), ( '<｜DSML｜parameter name="address" string="true"><｜DSML｜parameter name="street" string="true">Main St<｜DSML｜parameter name="city" string="true">New York', False, ), ( '<｜DSML｜parameter name="address" string="true">{"street": "Main St"}', False, ), ( '<｜DSML｜parameter name="address" string="false">{"city": "New York"}', False, ), ( '<｜DSML｜parameter name="address" string="true">{"street": "Main St", "city": "New York", "additional_property": "value"}<｜DSML｜parameter name="additional_property" string="true">value', True, ), ( '<｜DSML｜parameter name="address" string="true">{"street": "Main St", "city": "New York", "additional_property": value}', False, ), ) @pytest.mark.parametrize( "input_str, accepted", deepseek_test_inner_object_schema_input_str_accepted ) def test_deepseek_inner_object_schema(input_str: str, accepted: bool): expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "<｜DSML｜parameter name=\"" xml_variable_name "\" string=\"" ("true" | "false") "\">" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "<｜DSML｜parameter name=\"" xml_variable_name "\" string=\"" ("true" | "false") "\">" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_0_addl ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object root_prop_0_addl_key ::= ["] (("\"" | [^cs\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "c" ("\"" | [^i\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "i" ("\"" | [^t\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "t" ("\"" | [^y\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "y" ([^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub)))) | "s" ("\"" | [^t\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "t" ("\"" | [^r\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "r" ("\"" | [^e\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "e" ("\"" | [^e\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "e" ("\"" | [^t\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "t" ([^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub)))))))) (= [ \n\t]* [,}\]:]) root_prop_0_part_1 ::= ([ \n\t]* "," [ \n\t]* root_prop_0_addl_key [ \n\t]* ":" [ \n\t]* root_prop_0_addl)* root_prop_0_part_0 ::= [ \n\t]* "," [ \n\t]* "\"city\"" [ \n\t]* ":" [ \n\t]* basic_string root_prop_0_part_1 root_prop_0 ::= "{" [ \n\t]* (("\"street\"" [ \n\t]* ":" [ \n\t]* basic_string root_prop_0_part_0)) [ \n\t]* "}" root_addl ::= xml_string | basic_array | basic_object root_part_0 ::= ([ \n\t]* "<｜DSML｜parameter name=\"" xml_variable_name "\" string=\"" ("true" | "false") "\">" [ \n\t]* root_addl [ \n\t]* "")* root ::= [ \n\t]* (("<｜DSML｜parameter name=\"address\" string=\"" ("true" | "false") "\">" [ \n\t]* root_prop_0 [ \n\t]* "" root_part_0)) [ \n\t]* """ schema = { "type": "object", "properties": { "address": { "type": "object", "properties": {"street": {"type": "string"}, "city": {"type": "string"}}, "required": ["street", "city"], "additionalProperties": True, } }, "additionalProperties": True, "required": ["address"], } _check_deepseek_grammar(schema, expected_grammar, input_str, accepted) deepseek_test_numbers_schema_input_str_accepted = ( ('<｜DSML｜parameter name="age" string="false">25', False), ( '<｜DSML｜parameter name="name" string="true">Bob<｜DSML｜parameter name="age" string="false">25', True, ), ( '<｜DSML｜parameter name="name" string="true">Bob<｜DSML｜parameter name="ID" string="false">123456<｜DSML｜parameter name="is_student" string="true">true', True, ), ( '<｜DSML｜parameter name="name" string="true">John<｜DSML｜parameter name="age" string="false">1<｜DSML｜parameter name="ID" string="false">1<｜DSML｜parameter name="is_student" string="false">false', False, ), ) @pytest.mark.parametrize("input_str, accepted", deepseek_test_numbers_schema_input_str_accepted) def test_deepseek_numbers_schema(input_str: str, accepted: bool): expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "<｜DSML｜parameter name=\"" xml_variable_name "\" string=\"" ("true" | "false") "\">" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "<｜DSML｜parameter name=\"" xml_variable_name "\" string=\"" ("true" | "false") "\">" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_1 ::= ("0" | "-"? [1-9] [0-9]*) root_prop_2 ::= ("0" | "-"? [1-9] [0-9]*) root_prop_3 ::= "true" | "false" root_part_2_1 ::= [ \n\t]* "<｜DSML｜parameter name=\"is_student\" string=\"" ("true" | "false") "\">" [ \n\t]* root_prop_3 [ \n\t]* "" "" root_part_2_2 ::= "" | [ \n\t]* "<｜DSML｜parameter name=\"is_student\" string=\"" ("true" | "false") "\">" [ \n\t]* root_prop_3 [ \n\t]* "" "" root_part_2_3 ::= "" root_part_1_1 ::= root_part_2_1 | [ \n\t]* "<｜DSML｜parameter name=\"ID\" string=\"" ("true" | "false") "\">" [ \n\t]* root_prop_2 [ \n\t]* "" root_part_2_2 root_part_1_2 ::= root_part_2_2 | [ \n\t]* "<｜DSML｜parameter name=\"ID\" string=\"" ("true" | "false") "\">" [ \n\t]* root_prop_2 [ \n\t]* "" root_part_2_3 root_part_0_1 ::= root_part_1_1 | [ \n\t]* "<｜DSML｜parameter name=\"age\" string=\"" ("true" | "false") "\">" [ \n\t]* root_prop_1 [ \n\t]* "" root_part_1_2 root ::= [ \n\t]* (("<｜DSML｜parameter name=\"name\" string=\"" ("true" | "false") "\">" [ \n\t]* xml_string [ \n\t]* "" root_part_0_1) | ("<｜DSML｜parameter name=\"age\" string=\"" ("true" | "false") "\">" [ \n\t]* root_prop_1 [ \n\t]* "" root_part_1_1) | ("<｜DSML｜parameter name=\"ID\" string=\"" ("true" | "false") "\">" [ \n\t]* root_prop_2 [ \n\t]* "" root_part_2_1)) [ \n\t]* """ schema = { "type": "object", "properties": { "name": {"type": "string"}, "age": {"type": "integer"}, "ID": {"type": "integer"}, "is_student": {"type": "boolean"}, }, "maxProperties": 3, "minProperties": 2, } _check_deepseek_grammar(schema, expected_grammar, input_str, accepted) # DeepSeek: reject Qwen format , Minimax format (no string=), accept <｜DSML｜parameter name="key" string="true|false"> deepseek_reject_wrong_parameter_format_input_str_accepted = ( ("Bob100", False), # Qwen format ( 'Bob100', False, ), # Minimax format (no string=) ( '<｜DSML｜parameter name="name" string="true">Bob<｜DSML｜parameter name="age" string="false">100', True, ), # correct ) @pytest.mark.parametrize( "input_str, accepted", deepseek_reject_wrong_parameter_format_input_str_accepted ) def test_deepseek_reject_wrong_parameter_format(input_str: str, accepted: bool): """DeepSeek grammar must accept <｜DSML｜parameter name=\"key\" string=\"true|false\">, reject Qwen and Minimax formats.""" expected_grammar = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "<｜DSML｜parameter name=\"" xml_variable_name "\" string=\"" ("true" | "false") "\">" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "<｜DSML｜parameter name=\"" xml_variable_name "\" string=\"" ("true" | "false") "\">" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_1 ::= ("0" | "-"? [1-9] [0-9]*) root_part_0 ::= [ \n\t]* "<｜DSML｜parameter name=\"age\" string=\"" ("true" | "false") "\">" [ \n\t]* root_prop_1 [ \n\t]* "" "" root ::= [ \n\t]* (("<｜DSML｜parameter name=\"name\" string=\"" ("true" | "false") "\">" [ \n\t]* xml_string [ \n\t]* "" root_part_0)) [ \n\t]* """ schema = { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], } _check_deepseek_grammar(schema, expected_grammar, input_str, accepted) # ---------- GLM XML tool calling (_glm_xml_tool_calling_to_ebnf) ---------- # Format: $PARAMETER_NAME$PARAMETER_VALUE glm_reject_wrong_parameter_format_input_str_accepted = ( ("Bob100", False), ('Bob100', False), ( '<｜DSML｜parameter name="name" string="true">Bob<｜DSML｜parameter name="age" string="false">100', False, ), ( "nameBob" "age100", True, ), ) @pytest.mark.parametrize( "input_str, accepted", glm_reject_wrong_parameter_format_input_str_accepted ) def test_glm_reject_wrong_parameter_format(input_str: str, accepted: bool): """GLM grammar must use arg_key/arg_value wrappers and reject other XML styles.""" schema = { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], } ebnf_grammar = _glm_xml_tool_calling_to_ebnf(schema) grammar_str = str(ebnf_grammar) assert "" in grammar_str assert "" in grammar_str _check_glm_grammar(schema, input_str, accepted) def test_nested_true_schema(): schema = {"type": "object", "properties": {"name": True}, "required": ["name"]} ebnf_grammar = _qwen_xml_tool_calling_to_ebnf(schema) assert _is_grammar_accept_string(ebnf_grammar, "\nvalue\n") assert _is_grammar_accept_string(ebnf_grammar, "\n[1, 2, 3]\n") assert _is_grammar_accept_string( ebnf_grammar, '\n{"name": "Tom"}\n' ) assert not _is_grammar_accept_string(ebnf_grammar, "anything") def test_true_schema(): schema = "true" ebnf_grammar = _qwen_xml_tool_calling_to_ebnf(schema) assert _is_grammar_accept_string(ebnf_grammar, "\nvalue\n") assert _is_grammar_accept_string(ebnf_grammar, "\n[1, 2, 3]\n") assert _is_grammar_accept_string( ebnf_grammar, '\n{"name": "Tom"}\n' ) assert not _is_grammar_accept_string(ebnf_grammar, "anything") if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.2.3/tests/python/test_grammar_compiler.py000066400000000000000000000335521521764210300227570ustar00rootroot00000000000000"""This test uses the optimized JSON grammar provided by the grammar library.""" import sys import threading import time from typing import Dict, List, Tuple import pytest from pydantic import BaseModel from transformers import AutoTokenizer import xgrammar as xgr from xgrammar.testing import _get_allow_empty_rule_ids @pytest.mark.hf_token_required def test_compiled_grammar(): grammar = xgr.Grammar.builtin_json_grammar() tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-chat-hf") tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) time_start = time.monotonic_ns() context = compiler.compile_grammar(grammar) time_end = time.monotonic_ns() print(f"Time to get compiled grammar: {(time_end - time_start) / 1e3} us") def check_matcher(matcher: xgr.GrammarMatcher): assert not matcher.is_terminated() assert not matcher.accept_string('{ name: "John" }') assert matcher.accept_string('{"name": "John"}') assert matcher.is_terminated() time_start = time.monotonic_ns() matcher_1 = xgr.GrammarMatcher(context, terminate_without_stop_token=True) time_end = time.monotonic_ns() print(f"Time to init matcher 1: {(time_end - time_start) / 1e3} us") check_matcher(matcher_1) time_start = time.monotonic_ns() matcher_2 = xgr.GrammarMatcher(context, terminate_without_stop_token=True) time_end = time.monotonic_ns() print(f"Time to init matcher 2: {(time_end - time_start) / 1e3} us") check_matcher(matcher_2) # Test max_threads=1 since we have a special logic to avoid using ThreadPool and mutex @pytest.mark.hf_token_required @pytest.mark.parametrize("max_threads", (8, 1)) def test_grammar_compiler_json(max_threads): tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-chat-hf") tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) time_start = time.monotonic_ns() grammar_compiler = xgr.GrammarCompiler(tokenizer_info, max_threads=max_threads) time_end = time.monotonic_ns() print(f"Time to init cached grammar compiler: {(time_end - time_start) / 1e3} us") def check_matcher(matcher: xgr.GrammarMatcher): assert not matcher.is_terminated() assert not matcher.accept_string('{ name: "John" }') assert matcher.accept_string('{"name": "John"}') assert matcher.is_terminated() time_start = time.monotonic_ns() compiled_grammar = grammar_compiler.compile_builtin_json_grammar() time_end = time.monotonic_ns() print(f"Time to get compiled grammar: {(time_end - time_start) / 1e3} us") matcher = xgr.GrammarMatcher(compiled_grammar, terminate_without_stop_token=True) check_matcher(matcher) time_start = time.monotonic_ns() compiled_grammar = grammar_compiler.compile_builtin_json_grammar() time_end = time.monotonic_ns() print(f"Time to get compiled grammar again: {(time_end - time_start) / 1e3} us") matcher = xgr.GrammarMatcher(compiled_grammar, terminate_without_stop_token=True) check_matcher(matcher) grammar_compiler.clear_cache() time_start = time.monotonic_ns() compiled_grammar = grammar_compiler.compile_builtin_json_grammar() time_end = time.monotonic_ns() print(f"Time to get compiled grammar after clear: {(time_end - time_start) / 1e3} us") matcher = xgr.GrammarMatcher(compiled_grammar, terminate_without_stop_token=True) check_matcher(matcher) @pytest.mark.hf_token_required def test_grammar_compiler_json_schema(): tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-chat-hf") tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) grammar_compiler = xgr.GrammarCompiler(tokenizer_info) class MainModel(BaseModel): integer_field: int number_field: float boolean_field: bool any_array_field: List array_field: List[str] tuple_field: Tuple[str, int, List[str]] object_field: Dict[str, int] nested_object_field: Dict[str, Dict[str, int]] instance = MainModel( integer_field=42, number_field=3.14e5, boolean_field=True, any_array_field=[3.14, "foo", None, True], array_field=["foo", "bar"], tuple_field=("foo", 42, ["bar", "baz"]), object_field={"foo": 42, "bar": 43}, nested_object_field={"foo": {"bar": 42}}, ) def check_with_fmt(any_whitespace, indent, separators, test_id): instance_str = instance.model_dump_json(indent=indent, round_trip=True) time_start = time.monotonic_ns() compiled_grammar = grammar_compiler.compile_json_schema( MainModel, any_whitespace=any_whitespace, indent=indent, separators=separators ) time_end = time.monotonic_ns() print(f"Time to get compiled grammar {test_id}: {(time_end - time_start) / 1e3} us") matcher = xgr.GrammarMatcher(compiled_grammar, terminate_without_stop_token=True) assert not matcher.is_terminated() assert matcher.accept_string(instance_str) assert matcher.is_terminated() check_with_fmt(False, None, (",", ":"), "1") check_with_fmt(False, None, (",", ":"), "2") check_with_fmt(False, 2, None, "3") check_with_fmt(False, 2, (",", ": "), "4") check_with_fmt(True, None, (",", ":"), "5") check_with_fmt(True, None, (",", ":"), "6") check_with_fmt(True, 2, None, "7") check_with_fmt(True, 2, (",", ": "), "8") grammar_compiler.clear_cache() check_with_fmt(False, None, (",", ":"), "9") grammar_expected_test_get_allow_empty_rule_ids = [ ( r"""root ::= rule1 rule2 | "abc" rule1 ::= "abc" | "" rule2 ::= "def" rule3 | "" rule3 ::= "ghi" """, [0, 1, 2], ), ( r"""root ::= rule1 rule2 [a-z]* rule1 ::= "abc" | "" rule2 ::= "def" | "" """, [0, 1, 2], ), ( r"""root ::= rule1 rule3 rule1 ::= "abc" | "" rule2 ::= "def" | "" rule3 ::= rule1 rule2 """, [0, 1, 2, 3], ), ( r"""root ::= [a]* [b]* rule1 rule1 ::= [abc]* [def]* """, [0, 1], ), ] @pytest.mark.parametrize("grammar, expected", grammar_expected_test_get_allow_empty_rule_ids) def test_get_allow_empty_rule_ids(grammar: str, expected: List[int]): grammar_compiler = xgr.GrammarCompiler(xgr.TokenizerInfo([])) compiled_grammar = grammar_compiler.compile_grammar(grammar) allow_empty_rule_ids = _get_allow_empty_rule_ids(compiled_grammar) assert allow_empty_rule_ids == expected schema_instances = [ ( '{"type": "object","properties":{"username":{"type": "string"}},"required":["username"]}', '{"username":"Alice"}', ), ( '{"type": "object","properties":{"age":{"type": "integer"}},"required":["age"]}', '{"age":30}', ), ( '{"type": "object","properties":{"city":{"type": "string"}},"required":["city"]}', '{"city":"Paris"}', ), ( '{"type": "object","properties":{"isActive":{"type": "boolean"}},"required":["isActive"]}', '{"isActive":true}', ), ( '{"type": "object","properties":{"rating":{"type": "number"}},"required":["rating"]}', '{"rating":4.5}', ), ( '{"type": "object","properties":{"name":{"type": "string"}},"required":["name"]}', '{"name":"Bob"}', ), ( '{"type": "object","properties":{"quantity":{"type": "integer"}},"required":["quantity"]}', '{"quantity":10}', ), ( '{"type": "object","properties":{"color":{"type": "string"}},"required":["color"]}', '{"color":"blue"}', ), ( '{"type": "object","properties":{"temperature":{"type": "number"}},"required":["temperature"]}', '{"temperature":22.5}', ), ( '{"type": "object","properties":{"isCompleted":{"type": "boolean"}},"required":["isCompleted"]}', '{"isCompleted":false}', ), ] @pytest.mark.hf_token_required def test_grammar_compiler_json_schema_concurrent(): tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-chat-hf") tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) grammar_compiler = xgr.GrammarCompiler(tokenizer_info) def check_matcher(matcher: xgr.GrammarMatcher, instance_str: str): assert not matcher.is_terminated() assert matcher.accept_string(instance_str) assert matcher.is_terminated() num_schemas = len(schema_instances) thread_cnt = 100 threads = [] def compile_grammar(id: int, schema: str, instance_str: str): schema_id = id % num_schemas time_mid = time.monotonic_ns() print(f"Thread {id} start compile grammar {schema_id}: {(time_mid - time_start) / 1e3} us") compiled_grammar = grammar_compiler.compile_json_schema( schema, indent=None, separators=(",", ":"), strict_mode=True ) time_end = time.monotonic_ns() print(f"Thread {id} end compile grammar {schema_id}: {(time_end - time_start) / 1e3} us") matcher = xgr.GrammarMatcher(compiled_grammar, terminate_without_stop_token=True) check_matcher(matcher, instance_str) time_start = time.monotonic_ns() for i in range(thread_cnt): t = threading.Thread(target=compile_grammar, args=(i, *schema_instances[i % num_schemas])) threads.append(t) t.start() for t in threads: t.join() @pytest.mark.hf_token_required def test_grammar_compiler_cache_unlimited(): tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-3.1-8B-Instruct") tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) def make_schema(name_str: str): return { "properties": {name_str: {"type": "string"}}, "required": [name_str], "type": "object", } MB = 1024 * 1024 # Default no limit grammar_compiler = xgr.GrammarCompiler(tokenizer_info) assert grammar_compiler.cache_limit_bytes == -1 # No limit (default, -1) assert grammar_compiler.get_cache_size_bytes() == 0 # No memory usage sum_single = 0 for i in range(10): schema = make_schema(f"name_{i}") compiled_grammar = grammar_compiler.compile_json_schema(schema, strict_mode=True) sum_single += compiled_grammar.memory_size_bytes memory_usage = grammar_compiler.get_cache_size_bytes() print(f"Cache memory usage after {i + 1} schemas: {memory_usage / MB:.3f} MB / unlimited") old_size = grammar_compiler.get_cache_size_bytes() grammar_compiler.compile_json_schema(make_schema("name_0"), strict_mode=True) assert grammar_compiler.get_cache_size_bytes() == old_size @pytest.mark.hf_token_required def test_grammar_compiler_cache_limited(): tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-3.1-8B-Instruct") tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) def make_schema(name_str: str): return { "properties": {name_str: {"type": "string"}}, "required": [name_str], "type": "object", } MB = 1024 * 1024 # with a 2MB limit limit = int(2 * MB) grammar_compiler = xgr.GrammarCompiler(tokenizer_info, cache_limit_bytes=limit) assert grammar_compiler.cache_limit_bytes == limit assert grammar_compiler.get_cache_size_bytes() == 0 sum_single = 0 for i in range(10): schema = make_schema(f"name_{i}") compiled_grammar = grammar_compiler.compile_json_schema(schema, strict_mode=True) sum_single += compiled_grammar.memory_size_bytes memory_usage = grammar_compiler.get_cache_size_bytes() print( f"Cache memory usage after {i + 1} schemas: {memory_usage / MB:.3f} MB / {limit / MB:.3f} MB" ) # Test clear_cache grammar_compiler.clear_cache() assert grammar_compiler.get_cache_size_bytes() == 0 @pytest.mark.hf_token_required def test_grammar_compiler_crossing_cache_same_grammar(): grammar = xgr.Grammar.builtin_json_grammar() tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-chat-hf") tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) time_start = time.monotonic_ns() contexta = compiler.compile_grammar(grammar) time_end = time.monotonic_ns() print(f"Compile time: {(time_end - time_start) / 1e6} ms") compiler = xgr.GrammarCompiler(tokenizer_info) time_start = time.monotonic_ns() contextb = compiler.compile_grammar(grammar) time_end = time.monotonic_ns() print(f"Compile time: {(time_end - time_start) / 1e6} ms") assert contexta.serialize_json() == contextb.serialize_json() @pytest.mark.hf_token_required def test_grammar_compiler_crossing_cache_different_grammar_with_same_fsm(): grammar_a = """ root ::= "{" string "}" string ::= "\\"" [^"]* "\\"" | "'" [^']* "'" """ grammar_b = """ root ::= "[" string "]" string ::= "\\"" [^"]* "\\"" | "'" [^']* "'" """ tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-chat-hf") tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) time_start = time.monotonic_ns() _ = compiler.compile_grammar(grammar_a) time_end = time.monotonic_ns() print(f"Grammar A compiled in {(time_end - time_start) / 1e6} ms") time_start = time.monotonic_ns() contextb = compiler.compile_grammar(grammar_b) time_end = time.monotonic_ns() print(f"Grammar B compiled in {(time_end - time_start) / 1e6} ms") compiler.clear_cache() time_start = time.monotonic_ns() contextb_without_cache = compiler.compile_grammar(grammar_b) time_end = time.monotonic_ns() print(f"Grammar B recompiled in {(time_end - time_start) / 1e6} ms") assert ( contextb.serialize_json() == contextb_without_cache.serialize_json() ), "Cached and non-cached compilations should yield the same result." if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.2.3/tests/python/test_grammar_matcher_basic.py000066400000000000000000001117771521764210300237370ustar00rootroot00000000000000"""Test the basic functionality of GrammarMatcher.""" import math import random import sys from typing import List, Optional, Union import pytest import torch from transformers import AutoTokenizer import xgrammar as xgr from xgrammar.testing import ( _get_masked_tokens_from_bitmask, _get_matcher_from_grammar, _get_matcher_from_grammar_and_tokenizer_info, _is_grammar_accept_string, ) _is_cuda_available = torch.cuda.is_available() json_grammar = xgr.Grammar.builtin_json_grammar() grammar__input__accepted__test_accept_string = [ ("""root ::= [^a]+""", "bbb", True), ("""root ::= [^a]+""", "bba", False), ("""root ::= [^a]+""", "©", True), ("""root ::= [^a]+""", b"\xe2\xa1\xa1", True), ("""root ::= [^a]+""", b"\xe2\xa1\xa1\xa1", False), ("""root ::= [^a]+""", b"\xe2\xa1\xe2\xa1", False), ] @pytest.mark.parametrize("grammar, input, accepted", grammar__input__accepted__test_accept_string) def test_accept_string(grammar: str, input: Union[str, bytes], accepted: bool): matcher = _get_matcher_from_grammar(grammar) assert matcher.accept_string(input) == accepted input_accepted = ['{"name": "John"}', '{ "name" : "John" }'] @pytest.mark.parametrize("input_accepted", input_accepted) def test_grammar_accept(input_accepted: str): assert _is_grammar_accept_string(json_grammar, input_accepted) input_refused = ('{ name: "John" }', '{ "name": "John" } ') @pytest.mark.parametrize("input_refused", input_refused) def test_grammar_refuse(input_refused: str): assert not _is_grammar_accept_string(json_grammar, input_refused) def test_debug_print_internal_state(): matcher = _get_matcher_from_grammar(json_grammar) input_str = '{"name": "John"}' for c in input_str: assert matcher.accept_string(c) internal_state = matcher._debug_print_internal_state() assert len(internal_state) > 0 tokenizer_path__input_str__expected_rejected_sizes = [ ( "meta-llama/Llama-2-7b-chat-hf", '{"id": 1,"name": "Example"}', [ # fmt: off 31989, 31912, 270, 270, 270, 31973, 31846, 31846, 31948, 31915, 270, 270, 270, 270, 270, 31973, 31846, 31846, 263, 263, 263, 263, 263, 263, 263, 263, 31974, 31999, # fmt: on ], ), ( # test for llama 3 "meta-llama/Meta-Llama-3-8B-Instruct", '{"id": 1,"name": "Example哈哈"}', [ # fmt: off 128235, 127497, 4744, 4744, 4744, 127849, 126399, 126399, 126760, 127499, 4744, 4744, 4744, 4744, 4744, 127849, 126399, 126399, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 128066, 128111, 4694, 128066, 128111, 4694, 127873, 128255, # fmt: on ], ), ] @pytest.mark.hf_token_required @pytest.mark.parametrize( "tokenizer_path, input_str, expected_rejected_sizes", tokenizer_path__input_str__expected_rejected_sizes, ) def test_fill_next_token_bitmask( tokenizer_path: str, input_str: str, expected_rejected_sizes: Optional[List[int]] ): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) matcher = _get_matcher_from_grammar_and_tokenizer_info(json_grammar, tokenizer_info) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) input_bytes = input_str.encode("utf-8") rejected_sizes = [] for i, c in enumerate(input_bytes): matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask( token_bitmask, tokenizer_info.vocab_size ) rejected_sizes.append(len(rejected_token_ids)) if expected_rejected_sizes is not None: assert rejected_sizes[-1] == expected_rejected_sizes[i], ( rejected_sizes[-1], expected_rejected_sizes[i], ) assert matcher.accept_string(bytes([c])) matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size) rejected_sizes.append(len(rejected_token_ids)) if expected_rejected_sizes is not None: assert rejected_sizes[-1] == expected_rejected_sizes[-1] def test_token_operations(): """Test accepting token and finding the next token mask.""" vocab = [ # fmt: off "", "", "a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", "\n", " ", '"a":true', # fmt: on ] input_splitted = ["{", '"', "abc", 'b"', ":", "6", ", ", " ", '"a":true', "}"] input_ids = [vocab.index(t) for t in input_splitted] tokenizer_info = xgr.TokenizerInfo(vocab) matcher = _get_matcher_from_grammar_and_tokenizer_info(json_grammar, tokenizer_info) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) expected = [ ["{"], ['"', "}", "\n", " ", '"a":true'], ["", "a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", " "], ["", "a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", " "], [":", "\n", " ", ':"'], ['"', "{", "6", "\n", " "], ["}", ", ", "6", "\n", " "], [" ", "\n", '"', '"a":true'], [" ", "\n", '"', '"a":true'], ["}", ", ", "\n", " "], [""], ] result = [] for id in input_ids: matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask( token_bitmask, tokenizer_info.vocab_size ) accepted = list(set(range(len(vocab))) - set(rejected_token_ids)) accepted_tokens = [vocab[i] for i in accepted] result.append(accepted_tokens) assert id in accepted, vocab[id] assert matcher.accept_token(id) matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size) accepted = list(set(range(len(vocab))) - set(rejected_token_ids)) accepted_tokens = [vocab[i] for i in accepted] result.append(accepted_tokens) assert result == expected def test_rollback(): vocab = [ # fmt: off "", "", "a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", "\n", " ", '"a":true', # fmt: on ] input_splitted = ["{", '"', "abc", 'b"', ":", "6", ", ", " ", '"a":true', "}"] input_ids = [vocab.index(t) for t in input_splitted] tokenizer_info = xgr.TokenizerInfo(vocab) matcher = _get_matcher_from_grammar_and_tokenizer_info( json_grammar, tokenizer_info, max_rollback_tokens=5 ) assert matcher.max_rollback_tokens == -1 input_ids_splitted = [input_ids[i : i + 2] for i in range(0, len(input_ids), 2)] for i_1, i_2 in input_ids_splitted: orig_result = [] token_bitmask1 = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(token_bitmask1) orig_result.append(token_bitmask1) assert matcher.accept_token(i_1) token_bitmask2 = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(token_bitmask2) orig_result.append(token_bitmask2) assert matcher.accept_token(i_2) matcher.rollback(2) result_after_rollback = [] new_token_bitmask1 = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(new_token_bitmask1) result_after_rollback.append(new_token_bitmask1) assert matcher.accept_token(i_1) new_token_bitmask2 = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(new_token_bitmask2) result_after_rollback.append(new_token_bitmask2) assert matcher.accept_token(i_2) for l, r in zip(orig_result, result_after_rollback): torch.testing.assert_close(l, r) def test_graceful_rollback_failure(): vocab = [ # fmt: off "", "", "a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", "6:", ":", "\n", " ", '"a":true', # fmt: on ] input_splitted = ["{", '"', "abc", '"', ":"] input_ids = [vocab.index(t) for t in input_splitted] tokenizer_info = xgr.TokenizerInfo(vocab) matcher = _get_matcher_from_grammar_and_tokenizer_info( json_grammar, tokenizer_info, max_rollback_tokens=5 ) for i in input_ids: assert matcher.accept_token(i) assert not matcher.accept_token(vocab.index("6:")) # The matching should have accepted char '6' but failed to accept char ':' # A graceful revert should then occur, where char '6' is rolled back and # the state of the matcher is the same as before the failed call to accept_token for i in map(vocab.index, ['"', "abc", '"', " ", "}"]): assert matcher.accept_token(i) def test_reset(): vocab = [ # fmt: off "", "", "a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", "\n", " ", '"a":true', # fmt: on ] input_splitted = ["{", '"', "abc", 'b"', ":", "6", ", ", " ", '"a":true', "}"] input_ids = [vocab.index(t) for t in input_splitted] tokenizer_info = xgr.TokenizerInfo(vocab) matcher = _get_matcher_from_grammar_and_tokenizer_info(json_grammar, tokenizer_info) orig_result = [] for i in input_ids: token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(token_bitmask) orig_result.append(token_bitmask) assert matcher.accept_token(i) matcher.reset() result_after_reset = [] for i in input_ids: token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(token_bitmask) result_after_reset.append(token_bitmask) assert matcher.accept_token(i) for l, r in zip(orig_result, result_after_reset): torch.testing.assert_close(l, r) def test_termination(): vocab = [ # fmt: off "", "", "a", "abc", 'b"', '"', ':"', "{", " }", ", ", "6", ":", "\n", " ", '"a"', ':true', # fmt: on ] input_splitted = ["{", '"', "abc", 'b"', ":", "6", ", ", " ", '"a"', ":true", " }", ""] input_ids = [vocab.index(t) for t in input_splitted] tokenizer_info = xgr.TokenizerInfo(vocab) matcher = _get_matcher_from_grammar_and_tokenizer_info( json_grammar, tokenizer_info, max_rollback_tokens=5 ) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) for i in input_ids: matcher.fill_next_token_bitmask(token_bitmask) assert matcher.accept_token(i) assert matcher.is_terminated() assert matcher.accept_token(0) is False with pytest.raises(RuntimeError): matcher.fill_next_token_bitmask(token_bitmask) matcher.rollback(2) assert not matcher.is_terminated() assert matcher.accept_token(input_ids[-2]) def test_is_completed(): vocab = [ # fmt: off "", "", "a", "abc", 'b"', '"', ':"', "{", " }", ", ", "6", ":", "\n", " ", '"a"', ':true', # fmt: on ] # Input for a complete JSON object *without* the stop token input_without_stop = ["{", '"', "abc", 'b"', ":", "6", ", ", " ", '"a"', ":true", " }"] input_ids_without_stop = [vocab.index(t) for t in input_without_stop] stop_token_id = vocab.index("") tokenizer_info = xgr.TokenizerInfo(vocab) # --- Case 1: default mode (terminate_without_stop_token=False) --- matcher = _get_matcher_from_grammar_and_tokenizer_info( json_grammar, tokenizer_info, max_rollback_tokens=5 ) # Before any input: not completed, not terminated assert not matcher.is_completed() assert not matcher.is_terminated() # Feed tokens for a complete JSON object (no stop token yet) for i in input_ids_without_stop: assert matcher.accept_token(i) # Completed (valid JSON) but not terminated (stop token not accepted) assert matcher.is_completed() assert not matcher.is_terminated() # Accept stop token assert matcher.accept_token(stop_token_id) assert matcher.is_completed() assert matcher.is_terminated() # Rollback the stop token: still completed, no longer terminated matcher.rollback(1) assert matcher.is_completed() assert not matcher.is_terminated() # Rollback further into mid-parse: neither completed nor terminated matcher.rollback(2) assert not matcher.is_completed() assert not matcher.is_terminated() # --- Case 2: terminate_without_stop_token=True --- matcher2 = _get_matcher_from_grammar_and_tokenizer_info( json_grammar, tokenizer_info, terminate_without_stop_token=True ) assert not matcher2.is_completed() assert not matcher2.is_terminated() for i in input_ids_without_stop: assert matcher2.accept_token(i) # In this mode, completed and terminated are the same assert matcher2.is_completed() assert matcher2.is_terminated() def test_fork_initial_state(): """Fork at initial state: forked matcher has same state and same next-token bitmask.""" vocab = ["", "", "a", "b"] tokenizer_info = xgr.TokenizerInfo(vocab) grammar = xgr.Grammar.from_ebnf('root ::= "a" "b"') original_matcher = _get_matcher_from_grammar_and_tokenizer_info(grammar, tokenizer_info) forked_matcher = original_matcher.fork() bitmask_original = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) bitmask_forked = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) original_matcher.fill_next_token_bitmask(bitmask_original) forked_matcher.fill_next_token_bitmask(bitmask_forked) torch.testing.assert_close(bitmask_original, bitmask_forked) assert not original_matcher.is_terminated() and not forked_matcher.is_terminated() assert original_matcher.stop_token_ids == forked_matcher.stop_token_ids def test_fork_after_accept_tokens(): """Fork after accepting tokens: forked has same parsing state; both can then diverge.""" vocab = ["", "", "a", "abc", 'b"', '"', "{", "}", " ", ":"] tokenizer_info = xgr.TokenizerInfo(vocab) input_ids = [vocab.index(t) for t in ["{", '"', "abc", 'b"']] original_matcher = _get_matcher_from_grammar_and_tokenizer_info(json_grammar, tokenizer_info) for token_id in input_ids: assert original_matcher.accept_token(token_id) forked_matcher = original_matcher.fork() bitmask_original = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) bitmask_forked = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) original_matcher.fill_next_token_bitmask(bitmask_original) forked_matcher.fill_next_token_bitmask(bitmask_forked) torch.testing.assert_close(bitmask_original, bitmask_forked) next_token_id = vocab.index(":") assert original_matcher.accept_token(next_token_id) assert forked_matcher.accept_token(next_token_id) original_matcher.rollback(1) forked_matcher.rollback(1) original_matcher.fill_next_token_bitmask(bitmask_original) forked_matcher.fill_next_token_bitmask(bitmask_forked) torch.testing.assert_close(bitmask_original, bitmask_forked) def test_fork_after_rollback(): """Fork after rollback: forked state matches current state; original and forked independent.""" vocab = ["", "", "a", "b"] tokenizer_info = xgr.TokenizerInfo(vocab) grammar = xgr.Grammar.from_ebnf('root ::= "a" "b"') original_matcher = _get_matcher_from_grammar_and_tokenizer_info(grammar, tokenizer_info) assert original_matcher.accept_token(vocab.index("a")) assert original_matcher.accept_token(vocab.index("b")) original_matcher.rollback(1) forked_matcher = original_matcher.fork() bitmask_original = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) bitmask_forked = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) original_matcher.fill_next_token_bitmask(bitmask_original) forked_matcher.fill_next_token_bitmask(bitmask_forked) torch.testing.assert_close(bitmask_original, bitmask_forked) accepted_token_ids_forked = set(range(len(vocab))) - set( _get_masked_tokens_from_bitmask(bitmask_forked, len(vocab)) ) assert vocab.index("b") in accepted_token_ids_forked def test_fork_when_terminated(): """Fork when matcher is terminated: forked is also terminated; rollback on one is independent.""" vocab = ["", "", "a", "b"] tokenizer_info = xgr.TokenizerInfo(vocab) grammar = xgr.Grammar.from_ebnf('root ::= "a" "b"') original_matcher = _get_matcher_from_grammar_and_tokenizer_info(grammar, tokenizer_info) assert original_matcher.accept_token(vocab.index("a")) assert original_matcher.accept_token(vocab.index("b")) assert original_matcher.accept_token(vocab.index("")) assert original_matcher.is_terminated() forked_matcher = original_matcher.fork() assert forked_matcher.is_terminated() original_matcher.rollback(1) assert not original_matcher.is_terminated() assert forked_matcher.is_terminated() assert original_matcher.accept_token(vocab.index("")) def test_fork_independent_state(): """Original and forked evolve independently: accept on one does not change the other.""" vocab = ["", "", "a", "b", "c"] tokenizer_info = xgr.TokenizerInfo(vocab) grammar = xgr.Grammar.from_ebnf('root ::= "a" ("b" | "c")') original_matcher = _get_matcher_from_grammar_and_tokenizer_info(grammar, tokenizer_info) assert original_matcher.accept_token(vocab.index("a")) forked_matcher = original_matcher.fork() assert original_matcher.accept_token(vocab.index("b")) assert forked_matcher.accept_token(vocab.index("c")) bitmask_forked = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) forked_matcher.fill_next_token_bitmask(bitmask_forked) accepted_after_forked = set(range(len(vocab))) - set( _get_masked_tokens_from_bitmask(bitmask_forked, len(vocab)) ) assert accepted_after_forked == {vocab.index("")} assert original_matcher.accept_token(vocab.index("")) assert forked_matcher.accept_token(vocab.index("")) assert original_matcher.is_terminated() assert forked_matcher.is_terminated() def test_get_jump_forward_string(): grammar_ebnf = r"""root ::= "abb" | "abbd" | other_rule other_rule ::= "a" sub_rule "b" sub_rule ::= "b" """ grammar = xgr.Grammar.from_ebnf(grammar_ebnf) matcher = _get_matcher_from_grammar_and_tokenizer_info(grammar) assert matcher.accept_string("a") assert matcher.find_jump_forward_string() == "bb" def test_vocab_size(): vocab = [ # fmt: off "", "", "a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", "\n", " ", '"a":true', # fmt: on ] tokenizer_info = xgr.TokenizerInfo(vocab, vocab_size=64) matcher = _get_matcher_from_grammar_and_tokenizer_info(json_grammar, tokenizer_info) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(token_bitmask) assert token_bitmask.shape == (1, 2) rejected_tokens = _get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size) assert rejected_tokens == [i for i in range(64) if i != 7] tokenizer_path_override_stop_tokens = [ ("meta-llama/Llama-2-7b-chat-hf", [2]), ("meta-llama/Meta-Llama-3-8B-Instruct", [128001, 128009]), ("deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct", [100001]), ] @pytest.mark.hf_token_required @pytest.mark.parametrize( "tokenizer_path, override_stop_tokens", tokenizer_path_override_stop_tokens ) def test_override_stop_tokens(tokenizer_path: str, override_stop_tokens: List[int]): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info_1 = xgr.TokenizerInfo.from_huggingface( tokenizer, stop_token_ids=override_stop_tokens ) matcher_1 = _get_matcher_from_grammar_and_tokenizer_info(json_grammar, tokenizer_info_1) assert tokenizer_info_1.stop_token_ids == override_stop_tokens assert matcher_1.stop_token_ids == override_stop_tokens tokenizer_info_2 = xgr.TokenizerInfo.from_huggingface(tokenizer) matcher_2 = _get_matcher_from_grammar_and_tokenizer_info( json_grammar, tokenizer_info_2, override_stop_tokens=override_stop_tokens ) assert matcher_2.stop_token_ids == override_stop_tokens @pytest.mark.hf_token_required def test_fill_next_token_bitmask_errors(): # llama 3.1 8b tokenizer = AutoTokenizer.from_pretrained( "meta-llama/Meta-Llama-3-8B-Instruct", use_fast=True, trust_remote_code=True ) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) matcher = _get_matcher_from_grammar_and_tokenizer_info(json_grammar, tokenizer_info) bitmask1 = torch.zeros(1, math.ceil(tokenizer_info.vocab_size / 32) - 1, dtype=torch.int32) with pytest.raises(RuntimeError): matcher.fill_next_token_bitmask(bitmask1) bitmask2 = torch.zeros(1, math.ceil(tokenizer_info.vocab_size / 32), dtype=torch.int32) with pytest.raises(RuntimeError): matcher.fill_next_token_bitmask(bitmask2, index=1) bitmask3 = torch.zeros(1, math.ceil(tokenizer_info.vocab_size / 32), dtype=torch.float32) with pytest.raises(RuntimeError): matcher.fill_next_token_bitmask(bitmask3) if _is_cuda_available: bitmask3 = torch.zeros(1, math.ceil(tokenizer_info.vocab_size / 32), 1, dtype=torch.int32) with pytest.raises(RuntimeError): matcher.fill_next_token_bitmask(bitmask3) bitmask_correct = torch.zeros(1, math.ceil(tokenizer_info.vocab_size / 32), dtype=torch.int32) matcher.fill_next_token_bitmask(bitmask_correct) test_batch_accept_string_grammars_inputs_expecteds = [ (['root ::= "a"', "root ::= [0-9]+", 'root ::= "ab"'], ["a", b"123", "ab"], [True, True, True]), ( ['root ::= "a"', "root ::= [0-9]+", 'root ::= "ab"'], ["b", "123a", "d"], [False, False, False], ), ( ['root ::= "a"', "root ::= [0-9]+", 'root ::= "ab"'], ["a", b"123a", b"ab"], [True, False, True], ), (['root ::= "a"'], ["a"], [True]), (['root ::= "a"'], ["b"], [False]), ( ['root ::= "你好"', 'root ::= "こんにちは"', 'root ::= "안녕하세요"'], ["你好", "こんにちは", "안녕하세요"], [True, True, True], ), ] @pytest.mark.parametrize( "grammars, inputs, expecteds", test_batch_accept_string_grammars_inputs_expecteds ) def test_batch_accept_string( grammars: List[str], inputs: List[Union[str, bytes]], expecteds: List[bool] ): matchers = [_get_matcher_from_grammar(grammar) for grammar in grammars] results = xgr.BatchGrammarMatcher.batch_accept_string(matchers, inputs) assert results == expecteds test_batch_accept_token_grammars_inputs_expecteds = [ (['root ::= "a"', "root ::= [0-9]+", 'root ::= "ab"'], [2, 5, 2], [True, True, True]), (['root ::= "a"', "root ::= [0-9]+", 'root ::= "ab"'], [3, 2, 4], [False, False, False]), (['root ::= "a"', "root ::= [0-9]+", 'root ::= "ab"'], [2, 8, 9], [True, False, True]), (['root ::= "a"'], [2], [True]), (['root ::= "a"'], [3], [False]), ] @pytest.mark.parametrize( "grammars, inputs, expecteds", test_batch_accept_token_grammars_inputs_expecteds ) def test_batch_accept_token(grammars: List[str], inputs: List[int], expecteds: List[bool]): vocab = [ # fmt: off "", "", "a", "b", "c", "1", "2", "3", "123a", "ab", # fmt: on ] tokenizer_info = xgr.TokenizerInfo(vocab) matchers = [ _get_matcher_from_grammar_and_tokenizer_info(xgr.Grammar.from_ebnf(grammar), tokenizer_info) for grammar in grammars ] results = xgr.BatchGrammarMatcher.batch_accept_token(matchers, inputs) assert results == expecteds def test_batch_rollback(): """Batch rollback: 3 matchers with rollback lengths 0, 1, 2; re-accept yields same bitmasks.""" vocab = [ # fmt: off "", "", "a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", "\n", " ", '"a":true', # fmt: on ] input_splitted = ["{", '"', "abc", 'b"', ":", "6", ", ", " ", '"a":true', "}"] input_ids = [vocab.index(t) for t in input_splitted] tokenizer_info = xgr.TokenizerInfo(vocab) matchers = [ _get_matcher_from_grammar_and_tokenizer_info(json_grammar, tokenizer_info), _get_matcher_from_grammar_and_tokenizer_info(json_grammar, tokenizer_info), _get_matcher_from_grammar_and_tokenizer_info(json_grammar, tokenizer_info), ] rollback_lengths = [0, 1, 2] input_ids_pairs = [input_ids[i : i + 2] for i in range(0, len(input_ids), 2)] for first_token_id, second_token_id in input_ids_pairs: # Per matcher: bitmask_before_first_accept, bitmask_before_second_accept, bitmask_after_second_accept orig_bitmasks = [] for matcher in matchers: bitmask_before_first_accept = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(bitmask_before_first_accept) orig_bitmasks.append(bitmask_before_first_accept.clone()) assert matcher.accept_token(first_token_id) bitmask_before_second_accept = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(bitmask_before_second_accept) orig_bitmasks.append(bitmask_before_second_accept.clone()) assert matcher.accept_token(second_token_id) bitmask_after_second_accept = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(bitmask_after_second_accept) orig_bitmasks.append(bitmask_after_second_accept.clone()) xgr.BatchGrammarMatcher.batch_rollback(matchers, rollback_lengths) for matcher_index, matcher in enumerate(matchers): num_rollback = rollback_lengths[matcher_index] base = matcher_index * 3 if num_rollback == 0: bitmask_after_rollback = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(bitmask_after_rollback) torch.testing.assert_close(orig_bitmasks[base + 2], bitmask_after_rollback) elif num_rollback == 1: bitmask_after_rollback = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(bitmask_after_rollback) torch.testing.assert_close(orig_bitmasks[base + 1], bitmask_after_rollback) assert matcher.accept_token(second_token_id) bitmask_after_reaccept = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(bitmask_after_reaccept) torch.testing.assert_close(orig_bitmasks[base + 2], bitmask_after_reaccept) else: assert num_rollback == 2 bitmask_after_rollback = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(bitmask_after_rollback) torch.testing.assert_close(orig_bitmasks[base + 0], bitmask_after_rollback) assert matcher.accept_token(first_token_id) bitmask_after_first_reaccept = xgr.allocate_token_bitmask( 1, tokenizer_info.vocab_size ) matcher.fill_next_token_bitmask(bitmask_after_first_reaccept) torch.testing.assert_close(orig_bitmasks[base + 1], bitmask_after_first_reaccept) assert matcher.accept_token(second_token_id) bitmask_after_second_reaccept = xgr.allocate_token_bitmask( 1, tokenizer_info.vocab_size ) matcher.fill_next_token_bitmask(bitmask_after_second_reaccept) torch.testing.assert_close(orig_bitmasks[base + 2], bitmask_after_second_reaccept) def test_batch_rollback_single_matcher(): """Batch rollback with a single matcher (edge case).""" vocab = ["", "", "a", "b"] tokenizer_info = xgr.TokenizerInfo(vocab) grammar = xgr.Grammar.from_ebnf('root ::= "a" "b"') matcher = _get_matcher_from_grammar_and_tokenizer_info(grammar, tokenizer_info) assert matcher.accept_token(2) and matcher.accept_token(3) xgr.BatchGrammarMatcher.batch_rollback([matcher], [2]) next_token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(next_token_bitmask) accepted_token_ids = set(range(len(vocab))) - set( _get_masked_tokens_from_bitmask(next_token_bitmask, len(vocab)) ) assert accepted_token_ids == {2} # Only "a" allowed again assert matcher.accept_token(2) and matcher.accept_token(3) def test_batch_rollback_zero_and_mixed(): """Rollback 0 for some matchers and non-zero for others.""" vocab = ["", "", "a", "b", "c"] tokenizer_info = xgr.TokenizerInfo(vocab) grammar = xgr.Grammar.from_ebnf('root ::= "a" "b"') matcher_rolled_back = _get_matcher_from_grammar_and_tokenizer_info(grammar, tokenizer_info) matcher_unchanged = _get_matcher_from_grammar_and_tokenizer_info(grammar, tokenizer_info) # matcher_rolled_back: accept "a","b"; matcher_unchanged: accept "a" only assert matcher_rolled_back.accept_token(2) and matcher_rolled_back.accept_token(3) assert matcher_unchanged.accept_token(2) xgr.BatchGrammarMatcher.batch_rollback([matcher_rolled_back, matcher_unchanged], [1, 0]) # matcher_rolled_back rolled back 1 -> only "a" accepted; matcher_unchanged (0 rollback) -> still after "a". Both allow "b" bitmask_rolled_back = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) bitmask_unchanged = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher_rolled_back.fill_next_token_bitmask(bitmask_rolled_back) matcher_unchanged.fill_next_token_bitmask(bitmask_unchanged) accepted_token_ids_rolled_back = set(range(len(vocab))) - set( _get_masked_tokens_from_bitmask(bitmask_rolled_back, len(vocab)) ) accepted_token_ids_unchanged = set(range(len(vocab))) - set( _get_masked_tokens_from_bitmask(bitmask_unchanged, len(vocab)) ) assert 3 in accepted_token_ids_rolled_back and 3 in accepted_token_ids_unchanged assert matcher_rolled_back.accept_token(3) and matcher_unchanged.accept_token(3) def test_batch_rollback_size_mismatch(): """batch_rollback raises when len(matchers) != len(num_tokens).""" vocab = ["", "a"] tokenizer_info = xgr.TokenizerInfo(vocab) grammar = xgr.Grammar.from_ebnf('root ::= "a"') matchers = [ _get_matcher_from_grammar_and_tokenizer_info(grammar, tokenizer_info), _get_matcher_from_grammar_and_tokenizer_info(grammar, tokenizer_info), ] with pytest.raises(RuntimeError): xgr.BatchGrammarMatcher.batch_rollback(matchers, [1]) with pytest.raises(RuntimeError): xgr.BatchGrammarMatcher.batch_rollback(matchers, [1, 1, 1]) def test_batch_rollback_empty(): """batch_rollback with empty matchers and num_tokens is a no-op.""" xgr.BatchGrammarMatcher.batch_rollback([], []) def test_batch_fill_next_token_bitmask(): grammars = ['root ::= "a"', "root ::= [0-9]+", 'root ::= "ab"', "root ::= [a-z0-9]+"] vocab = [ # fmt: off "ab", "", "a", "b", "c", "1", "2", "3", "123a" # fmt: on ] tokenizer_info = xgr.TokenizerInfo(vocab) matchers = [ _get_matcher_from_grammar_and_tokenizer_info(xgr.Grammar.from_ebnf(grammar), tokenizer_info) for grammar in grammars ] batch_size = len(matchers) token_bitmask = xgr.allocate_token_bitmask(batch_size, tokenizer_info.vocab_size) input_str = ["a", "1", "a", "123a"] expected_accepted_tokens = [ [[2], [5, 6, 7], [0, 2], [0, 2, 3, 4, 5, 6, 7, 8]], [[1], [1, 5, 6, 7], [3], [0, 1, 2, 3, 4, 5, 6, 7, 8]], ] batch_grammar_matcher = xgr.BatchGrammarMatcher(2) batch_grammar_matcher.batch_fill_next_token_bitmask(matchers, token_bitmask) for i in range(batch_size): rejected_token_ids = _get_masked_tokens_from_bitmask( token_bitmask[i : i + 1], tokenizer_info.vocab_size ) accepted = list(set(range(len(vocab))) - set(rejected_token_ids)) accepted.sort() assert accepted == expected_accepted_tokens[0][i] assert xgr.BatchGrammarMatcher.batch_accept_string(matchers, input_str) == [ True, True, True, True, ] batch_grammar_matcher.batch_fill_next_token_bitmask(matchers, token_bitmask) for i in range(batch_size): rejected_token_ids = _get_masked_tokens_from_bitmask( token_bitmask[i : i + 1], tokenizer_info.vocab_size ) accepted = list(set(range(len(vocab))) - set(rejected_token_ids)) accepted.sort() assert accepted == expected_accepted_tokens[1][i] @pytest.mark.hf_token_required def test_batch_fill_next_token_bitmask_pressure(): tokenizer_path = "meta-llama/Llama-2-7b-chat-hf" input_str = '{"id": 1,"name": "Example"}' rejected_token_size = [ # fmt: off 31989, 31912, 270, 270, 270, 31973, 31846, 31846, 31948, 31915, 270, 270, 270, 270, 270, 31973, 31846, 31846, 263, 263, 263, 263, 263, 263, 263, 263, 31974, 31999, # fmt: on ] tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) matchers = [ _get_matcher_from_grammar_and_tokenizer_info(json_grammar, tokenizer_info) for _ in range(len(input_str) + 1) ] input_strs = [input_str[:i] for i in range(len(input_str))] + [input_str] xgr.BatchGrammarMatcher.batch_accept_string(matchers, input_strs) bitmask_2d = xgr.allocate_token_bitmask(len(matchers), tokenizer_info.vocab_size) batch_grammar_matcher = xgr.BatchGrammarMatcher(2) batch_grammar_matcher.batch_fill_next_token_bitmask(matchers, bitmask_2d) for i in range(len(matchers)): rejected_token_ids = _get_masked_tokens_from_bitmask( bitmask_2d[i], tokenizer_info.vocab_size ) assert len(rejected_token_ids) == rejected_token_size[i], ( i, len(rejected_token_ids), rejected_token_size[i], ) @pytest.mark.hf_token_required def test_batch_fill_next_token_bitmask_pressure_single_thread(): tokenizer_path = "meta-llama/Llama-2-7b-chat-hf" input_str = '{"id": 1,"name": "Example"}' rejected_token_size = [ # fmt: off 31989, 31912, 270, 270, 270, 31973, 31846, 31846, 31948, 31915, 270, 270, 270, 270, 270, 31973, 31846, 31846, 263, 263, 263, 263, 263, 263, 263, 263, 31974, 31999, # fmt: on ] tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) matchers = [ _get_matcher_from_grammar_and_tokenizer_info(json_grammar, tokenizer_info) for _ in range(len(input_str) + 1) ] input_strs = [input_str[:i] for i in range(len(input_str))] + [input_str] xgr.BatchGrammarMatcher.batch_accept_string(matchers, input_strs) bitmask_2d = xgr.allocate_token_bitmask(len(matchers), tokenizer_info.vocab_size) batch_grammar_matcher = xgr.BatchGrammarMatcher(1) batch_grammar_matcher.batch_fill_next_token_bitmask(matchers, bitmask_2d) for i in range(len(matchers)): rejected_token_ids = _get_masked_tokens_from_bitmask( bitmask_2d[i], tokenizer_info.vocab_size ) assert len(rejected_token_ids) == rejected_token_size[i], ( i, len(rejected_token_ids), rejected_token_size[i], ) @pytest.mark.hf_token_required def test_batch_fill_next_token_bitmask_pressure_shuffled(): tokenizer_path = "meta-llama/Llama-2-7b-chat-hf" input_str = '{"id": 1,"name": "Example"}' rejected_token_size = [ # fmt: off 31989, 31912, 270, 270, 270, 31973, 31846, 31846, 31948, 31915, 270, 270, 270, 270, 270, 31973, 31846, 31846, 263, 263, 263, 263, 263, 263, 263, 263, 31974, 31999, # fmt: on ] tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) matchers = [ _get_matcher_from_grammar_and_tokenizer_info(json_grammar, tokenizer_info) for _ in range(len(input_str) + 1) ] input_strs = [input_str[:i] for i in range(len(input_str))] + [input_str] xgr.BatchGrammarMatcher.batch_accept_string(matchers, input_strs) shuffled_indices = list(range(len(matchers))) random.shuffle(shuffled_indices) bitmask_2d = xgr.allocate_token_bitmask(len(matchers), tokenizer_info.vocab_size) batch_grammar_matcher = xgr.BatchGrammarMatcher() batch_grammar_matcher.batch_fill_next_token_bitmask(matchers, bitmask_2d, shuffled_indices) for i in range(len(matchers)): rejected_token_ids = _get_masked_tokens_from_bitmask( bitmask_2d[shuffled_indices[i]], tokenizer_info.vocab_size ) assert len(rejected_token_ids) == rejected_token_size[i], ( i, len(rejected_token_ids), rejected_token_size[i], ) if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.2.3/tests/python/test_grammar_matcher_ebnf.py000066400000000000000000001140611521764210300235550ustar00rootroot00000000000000"""This test is adopted from test_builtin_grammar_json.py, but the grammar is parsed from a unoptimized, non-simplified EBNF string. This is to test the robustness of the grammar matcher. """ import sys import time from typing import List import pytest import torch from transformers import AutoTokenizer import xgrammar as xgr from xgrammar.testing import ( _get_masked_tokens_from_bitmask, _get_matcher_from_grammar_and_tokenizer_info, _is_grammar_accept_string, _print_grammar_fsms, ) def test_simple(): grammar_str = """root ::= rule1 rule2 rule1 ::= (rule2 | rule3) "a" rule2 ::= "b" rule3 ::= "c" """ grammar = xgr.Grammar.from_ebnf(grammar_str) assert _is_grammar_accept_string(grammar, "bab") assert not _is_grammar_accept_string(grammar, "abb") assert _is_grammar_accept_string(grammar, "cab") input_accepted_test_repetition = ( ("aaa", True), ("abcbc", True), ("bcbcbcbcbc", True), ("bcbcbcbcbcbcbcb", True), ("d", False), ("aaaa", False), ) @pytest.mark.parametrize("input, accepted", input_accepted_test_repetition) def test_repetition(input: str, accepted: bool): grammar_str = """ root ::= rule {2, 3} rule ::= ("a" | [bc] {4,}) """ grammar = xgr.Grammar.from_ebnf(grammar_str) assert _is_grammar_accept_string(grammar, input) == accepted input_accepted_test_repetition_with_empty = ( ("aaa", True), ("abcbc", True), ("bcbcbcbcbc", True), ("bcbcbcbcbcbcbcb", True), ("aaaa", False), ("", True), ("a", True), ("d", True), ) @pytest.mark.parametrize("input, accepted", input_accepted_test_repetition_with_empty) def test_repetition_with_empty(input: str, accepted: bool): grammar_str = """ root ::= rule {2, 3} "d"? rule ::= ("a" | [bc] {4,}) | "" """ grammar = xgr.Grammar.from_ebnf(grammar_str) assert _is_grammar_accept_string(grammar, input) == accepted def test_utf8(): # Test utf8-encoded string with EBNF grammar ebnf_grammar_str = "root ::= [，]+" grammar = xgr.Grammar.from_ebnf(ebnf_grammar_str) accepted_inputs = ["，", "，，，", "，，，，，，，，，，，，，，，，，，，，，，"] for input_str in accepted_inputs: assert _is_grammar_accept_string(grammar, input_str, print_time=True) def test_custom_root_rule(): json_grammar_simple_ebnf = r""" root ::= basic_object basic_any ::= basic_string | basic_object basic_string ::= (([\"] basic_string_1 [\"])) basic_string_1 ::= "" | [^"\\\r\n] basic_string_1 | "\\" escape basic_string_1 escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_object ::= "{" ("" | ws basic_string ws ":" ws basic_any ( ws "," ws basic_string ws ":" ws basic_any)*) ws "}" ws ::= [ \n\t]* """ grammar = xgr.Grammar.from_ebnf(json_grammar_simple_ebnf, root_rule_name="basic_string") assert _is_grammar_accept_string(grammar, r'"abc\r\n"') assert not _is_grammar_accept_string(grammar, r'{"name": "John" }') json_grammar_ebnf = r""" root ::= basic_array | basic_object basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) ".0"? basic_number ::= ("0" | "-"? [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= (([\"] basic_string_1 [\"])) basic_string_1 ::= "" | [^"\\\x00-\x1F] basic_string_1 | "\\" escape basic_string_1 escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= "[" ("" | ws basic_any (ws "," ws basic_any)*) ws "]" basic_object ::= "{" ("" | ws basic_string ws ":" ws basic_any ( ws "," ws basic_string ws ":" ws basic_any)*) ws "}" ws ::= [ \n\t]* """ json_grammar = xgr.Grammar.from_ebnf(json_grammar_ebnf) json_input_accepted = [ '{"name": "John"}', '{ "name" : "John" }', "{}", "[]", '{"name": "Alice", "age": 30, "city": "New York"}', '{"name": "Mike", "hobbies": ["reading", "cycling", "hiking"]}', '{"name": "Emma", "address": {"street": "Maple Street", "city": "Boston"}}', '[{"name": "David"}, {"name": "Sophia"}]', ( '{"name": "William", "age": null, "married": true, "children": ["Liam", "Olivia"],' ' "hasPets": false}' ), ( '{"name": "Olivia", "contact": {"email": "olivia@example.com", "address": ' '{"city": "Chicago", "zipcode": "60601"}}}' ), ( '{"name": "Liam", "skills": ["Java", "Python"], "experience": ' '[{"company": "CompanyA", "years": 5}, {"company": "CompanyB", "years": 3}]}' ), ( '{"person": {"name": "Ethan", "age": 40}, "education": {"degree": "Masters", ' '"university": "XYZ University"}, "work": [{"company": "ABC Corp", "position": ' '"Manager"}, {"company": "DEF Corp", "position": "Senior Manager"}]}' ), ( '{"name": "Charlotte", "details": {"personal": {"age": 35, "hobbies": ["gardening", ' '"painting"]}, "professional": {"occupation": "Engineer", "skills": ' '["CAD", "Project Management"], "projects": [{"name": "Project A", ' '"status": "Completed"}, {"name": "Project B", "status": "In Progress"}]}}}' ), ] @pytest.mark.parametrize("json_input_accepted", json_input_accepted) def test_json_accept(json_input_accepted: str): assert _is_grammar_accept_string(json_grammar, json_input_accepted) json_input_refused = ( r'{ name: "John" }', r'{ "name": "John" } ', # trailing space is not accepted r'{ "name": "John", "age": 30, }', r'{ "name": "John", "address": { "street": "123 Main St", "city": "New York" }', r'{ "name": "John", "age": 30, "hobbies": ["reading", "traveling",], }', r'{ "name": "John", "age": 30.5.7 }', r'{ "name": "John, "age": 30, "hobbies": ["reading", "traveling"] }', ( r'{ "name": "John", "age": 30, "hobbies": ["reading", { "type": "outdoor", "list": ' r'["hiking", "swimming",]}] }' ), r'{ "name": "John", "age": 30, "status": "\P\J" }', ( r'{ "name": "John", "age": 30, "hobbies": ["reading", "traveling"], "address": ' r'{ "street": "123 Main St", "city": "New York", "coordinates": { "latitude": 40.7128, ' r'"longitude": -74.0060 }}}, "work": { "company": "Acme", "position": "developer" }}' ), ) @pytest.mark.parametrize("json_input_refused", json_input_refused) def test_json_refuse(json_input_refused: str): assert not _is_grammar_accept_string(json_grammar, json_input_refused) json_input_pressure = ( # Extra long string: 1k chars ( '["Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer nec odio. Praesent ' "libero. Sed cursus ante dapibus diam. Sed nisi. Nulla quis sem at nibh elementum " "imperdiet. Duis sagittis ipsum. Praesent mauris. Fusce nec tellus sed augue semper " "porta. Mauris massa. Vestibulum lacinia arcu eget nulla. Class aptent taciti sociosqu " "ad litora torquent per conubia nostra, per inceptos himenaeos. Curabitur sodales ligula " "in libero. Sed dignissim lacinia nunc. Curabitur tortor. Pellentesque nibh. Aenean quam. " "In scelerisque sem at dolor. Maecenas mattis. Sed convallis tristique sem. Proin ut " "ligula vel nunc egestas porttitor. Morbi lectus risus, iaculis vel, suscipit quis, " "luctus non, massa. Fusce ac turpis quis ligula lacinia aliquet. Mauris ipsum. Nulla " "metus metus, ullamcorper vel, tincidunt sed, euismod in, nibh. Quisque volutpat " "condimentum velit. Class aptent taciti sociosqu ad litora torquent per conubia nostra, " "per inceptos himenaeos. Nam nec ante. Sed lacinia, urna non tincidunt mattis, tortor " "neque adipiscing diam, a cursus ipsum ante quis turpis. Nulla facilisi. Ut fringilla. " "Suspendisse potenti. Nunc feugiat mi a tellus consequat imperdiet. Vestibulum sapien. " "Proin quam. Etiam ultrices. Suspendisse in justo eu magna luctus suscipit. Sed lectus. " "Integer euismod lacus luctus magna. Quisque cursus, metus vitae pharetra auctor, sem " 'massa mattis sem, at interdum magna augue eget diam."]' ), # long and complex json: 3k chars ( r"""{ "web-app": { "servlet": [ { "servlet-name": "cofaxCDS", "servlet-class": "org.cofax.cds.CDSServlet", "init-param": { "configGlossary:installationAt": "Philadelphia, PA", "configGlossary:adminEmail": "ksm@pobox.com", "configGlossary:poweredBy": "Cofax", "configGlossary:poweredByIcon": "/images/cofax.gif", "configGlossary:staticPath": "/content/static", "templateProcessorClass": "org.cofax.WysiwygTemplate", "templateLoaderClass": "org.cofax.FilesTemplateLoader", "templatePath": "templates", "templateOverridePath": "", "defaultListTemplate": "listTemplate.htm", "defaultFileTemplate": "articleTemplate.htm", "useJSP": false, "jspListTemplate": "listTemplate.jsp", "jspFileTemplate": "articleTemplate.jsp", "cachePackageTagsTrack": 200, "cachePackageTagsStore": 200, "cachePackageTagsRefresh": 60, "cacheTemplatesTrack": 100, "cacheTemplatesStore": 50, "cacheTemplatesRefresh": 15, "cachePagesTrack": 200, "cachePagesStore": 100, "cachePagesRefresh": 10, "cachePagesDirtyRead": 10, "searchEngineListTemplate": "forSearchEnginesList.htm", "searchEngineFileTemplate": "forSearchEngines.htm", "searchEngineRobotsDb": "WEB-INF/robots.db", "useDataStore": true, "dataStoreClass": "org.cofax.SqlDataStore", "redirectionClass": "org.cofax.SqlRedirection", "dataStoreName": "cofax", "dataStoreDriver": "com.microsoft.jdbc.sqlserver.SQLServerDriver", "dataStoreUrl": "jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon", "dataStoreUser": "sa", "dataStorePassword": "dataStoreTestQuery", "dataStoreTestQuery": "SET NOCOUNT ON;select test='test';", "dataStoreLogFile": "/usr/local/tomcat/logs/datastore.log", "dataStoreInitConns": 10, "dataStoreMaxConns": 100, "dataStoreConnUsageLimit": 100, "dataStoreLogLevel": "debug", "maxUrlLength": 500 } }, { "servlet-name": "cofaxEmail", "servlet-class": "org.cofax.cds.EmailServlet", "init-param": { "mailHost": "mail1", "mailHostOverride": "mail2" } }, { "servlet-name": "cofaxAdmin", "servlet-class": "org.cofax.cds.AdminServlet" }, { "servlet-name": "fileServlet", "servlet-class": "org.cofax.cds.FileServlet" }, { "servlet-name": "cofaxTools", "servlet-class": "org.cofax.cms.CofaxToolsServlet", "init-param": { "templatePath": "toolstemplates/", "log": 1, "logLocation": "/usr/local/tomcat/logs/CofaxTools.log", "logMaxSize": "", "dataLog": 1, "dataLogLocation": "/usr/local/tomcat/logs/dataLog.log", "dataLogMaxSize": "", "removePageCache": "/content/admin/remove?cache=pages&id=", "removeTemplateCache": "/content/admin/remove?cache=templates&id=", "fileTransferFolder": "/usr/local/tomcat/webapps/content/fileTransferFolder", "lookInContext": 1, "adminGroupID": 4, "betaServer": true } } ], "servlet-mapping": { "cofaxCDS": "/", "cofaxEmail": "/cofaxutil/aemail/*", "cofaxAdmin": "/admin/*", "fileServlet": "/static/*", "cofaxTools": "/tools/*" }, "taglib": { "taglib-uri": "cofax.tld", "taglib-location": "/WEB-INF/tlds/cofax.tld" } } }""" ), ) @pytest.mark.parametrize("json_input_pressure", json_input_pressure) def test_json_pressure(json_input_pressure: str): assert _is_grammar_accept_string(json_grammar, json_input_pressure, print_time=True) tokenizer_path__input_str__expected_rejected_sizes = [ ( # short test "meta-llama/Llama-2-7b-chat-hf", '{"id": 1,"name": "Example"}', [ # fmt: off 31989, 31912, 270, 270, 270, 31973, 31846, 31846, 31948, 31915, 270, 270, 270, 270, 270, 31973, 31846, 31846, 263, 263, 263, 263, 263, 263, 263, 263, 31974, 31999, # fmt: on ], ), ( # long test "meta-llama/Llama-2-7b-chat-hf", """{ "id": 1, "na": "ex", "ac": true, "t": ["t1", "t2"], "ne": {"lv2": {"val": "dp"}, "arr": [1, 2, 3]}, "res": "res" }""", [ # fmt: off 31989, 31912, 31912, 270, 270, 270, 31973, 31846, 31846, 31948, 31915, 31915, 270, 270, 270, 31973, 31846, 31846, 263, 263, 263, 31974, 31915, 31915, 270, 270, 270, 31973, 31846, 31846, 31997, 31997, 31998, 31974, 31915, 31915, 270, 270, 31973, 31846, 31846, 31840, 262, 262, 262, 31969, 31846, 31846, 262, 262, 262, 31969, 31974, 31915, 31915, 270, 270, 270, 31973, 31846, 31846, 31908, 270, 270, 270, 270, 31973, 31846, 31846, 31906, 270, 270, 270, 270, 31973, 31846, 31846, 262, 262, 262, 31968, 31970, 31915, 31915, 270, 270, 270, 270, 31973, 31846, 31846, 31840, 31943, 31846, 31846, 31943, 31846, 31846, 31943, 31970, 31974, 31915, 31915, 270, 270, 270, 270, 31973, 31846, 31846, 263, 263, 263, 263, 31974, 31974, 31999, # fmt: on ], ), ( # test for llama 3 "meta-llama/Meta-Llama-3-8B-Instruct", '{"id": 1,"name": "Example哈哈"}', [ # fmt: off 128235, 127497, 4744, 4744, 4744, 127849, 126399, 126399, 126760, 127499, 4744, 4744, 4744, 4744, 4744, 127849, 126399, 126399, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 128066, 128111, 4694, 128066, 128111, 4694, 127873, 128255, # fmt: on ], ), ] @pytest.mark.hf_token_required @pytest.mark.parametrize( "tokenizer_path, input_str, expected_rejected_sizes", tokenizer_path__input_str__expected_rejected_sizes, ) def test_fill_next_token_bitmask( tokenizer_path: str, input_str: str, expected_rejected_sizes: List[int] ): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) time_start = time.monotonic_ns() matcher = xgr.GrammarMatcher(compiler.compile_grammar(json_grammar_ebnf)) time_end = time.monotonic_ns() print(f"Time to init GrammarMatcher: {(time_end - time_start) / 1e3} us") token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) device = "cuda" if torch.cuda.is_available() else "cpu" logits_gpu = torch.zeros(tokenizer_info.vocab_size, dtype=torch.float32, device=device) input_bytes = input_str.encode("utf-8") for i, c in enumerate(input_bytes): # 1. fill_next_token_bitmask time_start = time.monotonic_ns() matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time to fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") # 2. Correctness verification rejected_token_ids = _get_masked_tokens_from_bitmask( token_bitmask, tokenizer_info.vocab_size ) assert len(rejected_token_ids) == expected_rejected_sizes[i] # 3. apply_token_bitmask_inplace if torch.cuda.is_available(): torch.cuda.synchronize() time_start = time.monotonic_ns() xgr.apply_token_bitmask_inplace(logits_gpu, token_bitmask.to(device)) if torch.cuda.is_available(): torch.cuda.synchronize() time_end = time.monotonic_ns() print(f"Time to apply_token_bitmask_inplace: {(time_end - time_start) / 1e3} us") # 4. accept_string print("Accepting char:", bytes([c])) time_start = time.monotonic_ns() assert matcher.accept_string(bytes([c])) time_end = time.monotonic_ns() print(f"Time to accept_token: {(time_end - time_start) / 1e3} us") # 5. Final correctness verification matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size) assert len(rejected_token_ids) == expected_rejected_sizes[-1] def test_nullable_grammar(): grammar_with_nullable_rules = """ root ::= rule1 | (rule1 rule1 rule1 rule3)+ rule1 ::= rule2 rule2 ::= [0-9]* rule3 ::= [a-z] """ test_string = ["abc12312398014a", ""] for s in test_string: assert _is_grammar_accept_string(grammar_with_nullable_rules, s) def test_predict_complete(): # Test complex prediction and completion with EBNF grammar. mixed_grammar_str = """root ::= rule1 [0-9]? rule1 ::= rule2 [0-9]? | rule4 [0-9]? rule2 ::= rule3 [0-9]? | rule2 [0-9]? | rule1 [0-9]? rule3 ::= rule4 [0-9]? | rule5 [0-9]? rule4 ::= rule5 [0-9]? | rule6 [0-9]? rule5 ::= rule6 [0-9]? | rule7 [0-9]? | rule8 [0-9]? rule6 ::= rule7 [0-9]? | rule1 [0-9]? rule7 ::= rule8 [0-9]? | rule9 [0-9]? rule8 ::= rule9 [0-9]? | rule7 [0-9]? rule9 ::= [0-9]? """ grammar = xgr.Grammar.from_ebnf(mixed_grammar_str) input_str = "" for i in range(10): assert _is_grammar_accept_string(grammar, input_str) input_str += "0" assert _is_grammar_accept_string(grammar, input_str) # Test right recursion right_recursion_grammar = "root ::= [a-z] root | [a-z]" accept_strings = ["a", "ab", "abc", "abcd", "abcde"] reject_strings = ["", "1", "a1", "ab1", "abc1"] for accept_string in accept_strings: assert _is_grammar_accept_string(right_recursion_grammar, accept_string) for reject_string in reject_strings: assert not _is_grammar_accept_string(right_recursion_grammar, reject_string) # Test the mixture of right recursion and other rules mixed_grammar_str = """root ::= rule1 rule1 ::= "{" rule2 | "" rule2 ::= root "}" """ test_strings = {"", "{}", "{{}}", "{{{}}}", "{{{{}}}}", "{{{{{}}}}}"} rejected_strings = {"{", "{}{}", "{{{{}", "{{}}}", "{{{{{}}}}}}"} for test_string in test_strings: assert _is_grammar_accept_string(mixed_grammar_str, test_string) for rejected_string in rejected_strings: assert not _is_grammar_accept_string(mixed_grammar_str, rejected_string) def test_advance(): # Test complex Advance and completion with EBNF grammar. ebnf_grammar_str = """root ::= rule1 rule1 ::= [a] | [a-b] | [a-c]* | "a" | "aaaaaaaaaaaaaaaaaaa" """ grammar = xgr.Grammar.from_ebnf(ebnf_grammar_str) for i in range(10): input_str = "a" * i assert _is_grammar_accept_string(grammar, input_str) def test_character_class_star_utf8(): ebnf_grammar_str = """root ::= [^0-9]*""" test_string = "worldせかい世界" assert _is_grammar_accept_string(ebnf_grammar_str, test_string) def test_positive_utf8_character_class_cyrillic(): """Test positive character class with Cyrillic UTF-8 range (2-byte sequences). Tests fix for issue #138: positive character classes with UTF-8 ranges like [а-я] should work correctly. """ # Cyrillic lowercase range а-я (U+0430 to U+044F) ebnf_grammar_str = "root ::= [а-я]+" grammar = xgr.Grammar.from_ebnf(ebnf_grammar_str) # Single Cyrillic character should be accepted assert _is_grammar_accept_string(grammar, "а") # U+0430 - first in range assert _is_grammar_accept_string(grammar, "я") # U+044F - last in range assert _is_grammar_accept_string(grammar, "п") # U+043F - middle of range # Multiple Cyrillic characters assert _is_grammar_accept_string(grammar, "привет") assert _is_grammar_accept_string(grammar, "абвгд") # Should reject non-matching characters assert not _is_grammar_accept_string(grammar, "hello") # ASCII assert not _is_grammar_accept_string(grammar, "123") # digits assert not _is_grammar_accept_string(grammar, "") # empty # Test uppercase Cyrillic range ebnf_grammar_upper = "root ::= [А-Я]+" grammar_upper = xgr.Grammar.from_ebnf(ebnf_grammar_upper) assert _is_grammar_accept_string(grammar_upper, "А") # U+0410 assert _is_grammar_accept_string(grammar_upper, "Я") # U+042F assert _is_grammar_accept_string(grammar_upper, "ПРИВЕТ") assert not _is_grammar_accept_string(grammar_upper, "привет") # lowercase # Test mixed Cyrillic range ebnf_grammar_mixed = "root ::= [а-яА-ЯёЁ]+" grammar_mixed = xgr.Grammar.from_ebnf(ebnf_grammar_mixed) assert _is_grammar_accept_string(grammar_mixed, "Привет") assert _is_grammar_accept_string(grammar_mixed, "ёлка") assert _is_grammar_accept_string(grammar_mixed, "ЁЖИК") def test_positive_utf8_character_class_cjk(): """Test positive character class with CJK UTF-8 range (3-byte sequences). Tests Chinese/Japanese/Korean characters which use 3-byte UTF-8 encoding. """ # CJK Unified Ideographs range (subset): 一-龥 (U+4E00 to U+9FA5) ebnf_grammar_str = "root ::= [一-龥]+" grammar = xgr.Grammar.from_ebnf(ebnf_grammar_str) # Single CJK character assert _is_grammar_accept_string(grammar, "一") # U+4E00 - first in range assert _is_grammar_accept_string(grammar, "中") # U+4E2D - middle assert _is_grammar_accept_string(grammar, "龥") # U+9FA5 - last in range # Multiple CJK characters assert _is_grammar_accept_string(grammar, "你好") assert _is_grammar_accept_string(grammar, "世界") assert _is_grammar_accept_string(grammar, "中文测试") # Should reject non-matching characters assert not _is_grammar_accept_string(grammar, "hello") # ASCII assert not _is_grammar_accept_string(grammar, "привет") # Cyrillic assert not _is_grammar_accept_string(grammar, "") # empty # Test Japanese Hiragana range: あ-ん (U+3041 to U+3093) ebnf_hiragana = "root ::= [あ-ん]+" grammar_hiragana = xgr.Grammar.from_ebnf(ebnf_hiragana) assert _is_grammar_accept_string(grammar_hiragana, "あ") # U+3041 assert _is_grammar_accept_string(grammar_hiragana, "ん") # U+3093 assert _is_grammar_accept_string(grammar_hiragana, "こんにちは") assert not _is_grammar_accept_string(grammar_hiragana, "漢字") # Kanji, not Hiragana def test_positive_utf8_character_class_emoji(): """Test positive character class with emoji UTF-8 range (4-byte sequences). Tests emoji characters which use 4-byte UTF-8 encoding (U+1F300 and above). """ # Emoji range: Miscellaneous Symbols and Pictographs (U+1F300 to U+1F5FF) # Note: Using a smaller range for reliable testing ebnf_grammar_str = "root ::= [😀-😿]+" # U+1F600 to U+1F63F (Emoticons) grammar = xgr.Grammar.from_ebnf(ebnf_grammar_str) # Single emoji assert _is_grammar_accept_string(grammar, "😀") # U+1F600 - first in range assert _is_grammar_accept_string(grammar, "😃") # U+1F603 - middle assert _is_grammar_accept_string(grammar, "😿") # U+1F63F - last in range # Multiple emojis assert _is_grammar_accept_string(grammar, "😀😃😄") # Should reject non-matching characters assert not _is_grammar_accept_string(grammar, "hello") # ASCII assert not _is_grammar_accept_string(grammar, "🌍") # Different emoji range assert not _is_grammar_accept_string(grammar, "") # empty def test_positive_utf8_character_class_mixed_ranges(): """Test positive character class with mixed UTF-8 byte-length ranges. Tests combining ASCII, 2-byte, 3-byte, and 4-byte UTF-8 characters. """ # Mix of ASCII, Cyrillic, and CJK ebnf_grammar_str = "root ::= [a-zа-я一-龥]+" grammar = xgr.Grammar.from_ebnf(ebnf_grammar_str) # Individual ranges assert _is_grammar_accept_string(grammar, "hello") # ASCII assert _is_grammar_accept_string(grammar, "привет") # Cyrillic assert _is_grammar_accept_string(grammar, "你好") # CJK # Mixed content assert _is_grammar_accept_string(grammar, "helloпривет你好") # Should reject uppercase ASCII and other characters assert not _is_grammar_accept_string(grammar, "HELLO") # Uppercase ASCII assert not _is_grammar_accept_string(grammar, "123") # digits def test_positive_utf8_single_char_class(): """Test positive character class with single UTF-8 character (not a range).""" # Single Cyrillic character (not a range) ebnf_grammar_str = "root ::= [а]+" grammar = xgr.Grammar.from_ebnf(ebnf_grammar_str) assert _is_grammar_accept_string(grammar, "а") assert _is_grammar_accept_string(grammar, "ааа") assert not _is_grammar_accept_string(grammar, "б") assert not _is_grammar_accept_string(grammar, "a") # ASCII 'a' is different from Cyrillic 'а' # Single CJK character ebnf_grammar_cjk = "root ::= [中]+" grammar_cjk = xgr.Grammar.from_ebnf(ebnf_grammar_cjk) assert _is_grammar_accept_string(grammar_cjk, "中") assert _is_grammar_accept_string(grammar_cjk, "中中中") assert not _is_grammar_accept_string(grammar_cjk, "国") @pytest.mark.hf_token_required def test_not_neighbour_character_class(): raw_grammar = "root ::= [a-cx-z]*" tokenizer_path = "meta-llama/Llama-2-7b-chat-hf" tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) grammar = xgr.Grammar.from_ebnf(raw_grammar) matcher = _get_matcher_from_grammar_and_tokenizer_info(grammar, tokenizer_info) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size) assert len(rejected_token_ids) == 31933 def test_nfa(): grammar_str = """ root ::= rule1 | rule2 | rule3 rule1 ::= "abc" | "" rule2 ::= "abd" | "" rule3 ::= [a-n] [b-c] "x" | "" """ grammar = xgr.Grammar.from_ebnf(grammar_str) assert _is_grammar_accept_string(grammar, "abc") assert _is_grammar_accept_string(grammar, "abx") assert _is_grammar_accept_string(grammar, "ccx") assert not _is_grammar_accept_string(grammar, "abb") assert not _is_grammar_accept_string(grammar, "ad") @pytest.mark.parametrize( "tokenizer_path,input_str,expected_rejected_sizes", [ ( "meta-llama/Llama-2-7b-chat-hf", # Input: "aбя中" - ASCII 'a', Cyrillic 'б' (2 bytes), 'я' (2 bytes), CJK '中' (3 bytes) "aбя中", # fmt: off [22129, 22128, 31984, 22128, 31984, 22128, 31992, 31936, 22128], # fmt: on ) ], ) @pytest.mark.hf_token_required def test_fill_next_token_bitmask_unicode_char_class( tokenizer_path: str, input_str: str, expected_rejected_sizes: List[int] ): """Test token bitmask generation for Unicode character classes. This test verifies that the grammar correctly handles mixed UTF-8 character classes (ASCII, Cyrillic, CJK) and produces consistent rejected token counts. """ tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) # Grammar with mixed UTF-8 character class (ASCII + Cyrillic + CJK) ebnf_grammar_str = "root ::= [a-zа-я一-龥]+" grammar = xgr.Grammar.from_ebnf(ebnf_grammar_str) time_start = time.monotonic_ns() matcher = xgr.GrammarMatcher(compiler.compile_grammar(grammar)) time_end = time.monotonic_ns() print(f"Time to init GrammarMatcher: {(time_end - time_start) / 1e3} us") token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) device = "cuda" if torch.cuda.is_available() else "cpu" logits_gpu = torch.zeros(tokenizer_info.vocab_size, dtype=torch.float32, device=device) input_bytes = input_str.encode("utf-8") for i, c in enumerate(input_bytes): # 1. fill_next_token_bitmask time_start = time.monotonic_ns() matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time to fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") # 2. Correctness verification rejected_token_ids = _get_masked_tokens_from_bitmask( token_bitmask, tokenizer_info.vocab_size ) assert len(rejected_token_ids) == expected_rejected_sizes[i], ( f"Byte {i} ({hex(c)}): expected {expected_rejected_sizes[i]} rejected, " f"got {len(rejected_token_ids)}" ) # 3. apply_token_bitmask_inplace if torch.cuda.is_available(): torch.cuda.synchronize() time_start = time.monotonic_ns() xgr.apply_token_bitmask_inplace(logits_gpu, token_bitmask.to(device)) if torch.cuda.is_available(): torch.cuda.synchronize() time_end = time.monotonic_ns() print(f"Time to apply_token_bitmask_inplace: {(time_end - time_start) / 1e3} us") # 4. accept_string print("Accepting char:", bytes([c])) time_start = time.monotonic_ns() assert matcher.accept_string(bytes([c])) time_end = time.monotonic_ns() print(f"Time to accept_token: {(time_end - time_start) / 1e3} us") # 5. Final correctness verification matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size) assert ( len(rejected_token_ids) == expected_rejected_sizes[-1] ), f"Final: expected {expected_rejected_sizes[-1]} rejected, got {len(rejected_token_ids)}" def test_positive_utf8_character_class_with_quantifier(): """Test positive character class with mixed UTF-8 ranges and quantifier. Tests the combination of ASCII, Cyrillic (2-byte), and CJK (3-byte) characters with a {0, 2048} quantifier to ensure proper handling of repeated UTF-8 matching. """ ebnf_grammar_str = "root ::= [a-zа-я一-龥]{0,2048}" grammar = xgr.Grammar.from_ebnf(ebnf_grammar_str) # Empty string should be accepted (min is 0) assert _is_grammar_accept_string(grammar, "") # Individual character types assert _is_grammar_accept_string(grammar, "hello") # ASCII assert _is_grammar_accept_string(grammar, "привет") # Cyrillic assert _is_grammar_accept_string(grammar, "你好世界") # CJK # Mixed content assert _is_grammar_accept_string(grammar, "helloпривет你好") assert _is_grammar_accept_string(grammar, "abc中文def") # Long strings within quantifier range assert _is_grammar_accept_string(grammar, "a" * 100) assert _is_grammar_accept_string(grammar, "я" * 100) assert _is_grammar_accept_string(grammar, "中" * 100) # Should reject uppercase ASCII and other characters assert not _is_grammar_accept_string(grammar, "HELLO") # Uppercase ASCII assert not _is_grammar_accept_string(grammar, "123") # digits assert not _is_grammar_accept_string(grammar, "hello!") # with special char def _assert_repeat_ref_active(grammar): """Compile the grammar and assert that RepeatRef edges exist in the FSM.""" tokenizer_info = xgr.TokenizerInfo([]) compiler = xgr.GrammarCompiler(tokenizer_info, cache_enabled=False) compiled = compiler.compile_grammar(grammar) fsm_str = _print_grammar_fsms(compiled.grammar) assert "Repeat(" in fsm_str, f"Expected RepeatRef edges in FSM, got:\n{fsm_str}" def test_repeat_ref_exact(): """Test exact repetition {200} that activates the kRepeatRef FSM path.""" grammar = xgr.Grammar.from_ebnf('root ::= "a"{200}') _assert_repeat_ref_active(grammar) assert _is_grammar_accept_string(grammar, "a" * 200) assert not _is_grammar_accept_string(grammar, "a" * 199) assert not _is_grammar_accept_string(grammar, "a" * 201) assert not _is_grammar_accept_string(grammar, "") def test_repeat_ref_unbounded(): """Test unbounded repetition {200,} that activates the kRepeatRef FSM path.""" grammar = xgr.Grammar.from_ebnf('root ::= "a"{200,}') _assert_repeat_ref_active(grammar) assert _is_grammar_accept_string(grammar, "a" * 200) assert _is_grammar_accept_string(grammar, "a" * 300) assert _is_grammar_accept_string(grammar, "a" * 1000) assert not _is_grammar_accept_string(grammar, "a" * 199) assert not _is_grammar_accept_string(grammar, "") def test_repeat_ref_range(): """Test range repetition {100,200} that activates the kRepeatRef FSM path.""" grammar = xgr.Grammar.from_ebnf('root ::= "a"{100,200}') _assert_repeat_ref_active(grammar) assert _is_grammar_accept_string(grammar, "a" * 100) assert _is_grammar_accept_string(grammar, "a" * 150) assert _is_grammar_accept_string(grammar, "a" * 200) assert not _is_grammar_accept_string(grammar, "a" * 99) assert not _is_grammar_accept_string(grammar, "a" * 201) def test_repeat_ref_boundary(): """Test that {128} does NOT activate RepeatRef, but {129} does.""" g128 = xgr.Grammar.from_ebnf('root ::= "a"{128}') tokenizer_info = xgr.TokenizerInfo([]) compiler = xgr.GrammarCompiler(tokenizer_info, cache_enabled=False) fsm128 = _print_grammar_fsms(compiler.compile_grammar(g128).grammar) assert "Repeat(" not in fsm128 assert _is_grammar_accept_string(g128, "a" * 128) assert not _is_grammar_accept_string(g128, "a" * 127) assert not _is_grammar_accept_string(g128, "a" * 129) g129 = xgr.Grammar.from_ebnf('root ::= "a"{129}') _assert_repeat_ref_active(g129) assert _is_grammar_accept_string(g129, "a" * 129) assert not _is_grammar_accept_string(g129, "a" * 128) assert not _is_grammar_accept_string(g129, "a" * 130) def test_repeat_ref_multichar_rule(): """Test RepeatRef with a multi-character rule body.""" grammar_str = """ root ::= item{200} item ::= "ab" """ grammar = xgr.Grammar.from_ebnf(grammar_str) _assert_repeat_ref_active(grammar) assert _is_grammar_accept_string(grammar, "ab" * 200) assert not _is_grammar_accept_string(grammar, "ab" * 199) assert not _is_grammar_accept_string(grammar, "ab" * 201) assert not _is_grammar_accept_string(grammar, "a" * 400) def test_repeat_ref_range_from_zero(): """Test range repetition {0,200} that activates the kRepeatRef FSM path.""" grammar = xgr.Grammar.from_ebnf('root ::= "a"{0,200}') _assert_repeat_ref_active(grammar) assert _is_grammar_accept_string(grammar, "") assert _is_grammar_accept_string(grammar, "a" * 1) assert _is_grammar_accept_string(grammar, "a" * 128) assert _is_grammar_accept_string(grammar, "a" * 200) assert not _is_grammar_accept_string(grammar, "a" * 201) def test_repeat_ref_nested_inner(): """Test repeat containing a rule with choices (repeat wraps complex rule).""" grammar_str = """ root ::= item{200} item ::= "a" | "b" """ grammar = xgr.Grammar.from_ebnf(grammar_str) _assert_repeat_ref_active(grammar) assert _is_grammar_accept_string(grammar, "a" * 200) assert _is_grammar_accept_string(grammar, "b" * 200) assert _is_grammar_accept_string(grammar, "ab" * 100) assert _is_grammar_accept_string(grammar, "a" * 100 + "b" * 100) assert not _is_grammar_accept_string(grammar, "a" * 199) assert not _is_grammar_accept_string(grammar, "a" * 201) assert not _is_grammar_accept_string(grammar, "c" * 200) def test_repeat_ref_nested_outer(): """Test repeat used as part of a larger sequence (other rules wrap repeat).""" grammar_str = """ root ::= "start-" body "-end" body ::= [a-z]{200} """ grammar = xgr.Grammar.from_ebnf(grammar_str) _assert_repeat_ref_active(grammar) assert _is_grammar_accept_string(grammar, "start-" + "a" * 200 + "-end") assert _is_grammar_accept_string(grammar, "start-" + "xyz" * 66 + "xy" + "-end") assert not _is_grammar_accept_string(grammar, "start-" + "a" * 199 + "-end") assert not _is_grammar_accept_string(grammar, "start-" + "a" * 201 + "-end") assert not _is_grammar_accept_string(grammar, "a" * 200) def test_repeat_ref_sequence_with_repeat(): """Test repeat adjacent to other elements in a sequence.""" grammar_str = """ root ::= prefix middle suffix prefix ::= "x"{129} middle ::= [0-9]{200} suffix ::= "y"{129} """ grammar = xgr.Grammar.from_ebnf(grammar_str) _assert_repeat_ref_active(grammar) assert _is_grammar_accept_string(grammar, "x" * 129 + "0" * 200 + "y" * 129) assert _is_grammar_accept_string(grammar, "x" * 129 + "1234567890" * 20 + "y" * 129) assert not _is_grammar_accept_string(grammar, "x" * 128 + "0" * 200 + "y" * 129) assert not _is_grammar_accept_string(grammar, "x" * 129 + "0" * 199 + "y" * 129) assert not _is_grammar_accept_string(grammar, "x" * 129 + "0" * 200 + "y" * 128) def test_repeat_ref_complex_nested(): """Test deeply nested structure: repeat of sequence containing repeat and choices.""" grammar_str = """ root ::= "[" row{150} "]" row ::= "(" cell{130} ")" sep cell ::= [a-c] sep ::= "," | "" """ grammar = xgr.Grammar.from_ebnf(grammar_str) _assert_repeat_ref_active(grammar) single_row = "(" + "a" * 130 + ")" # 150 rows: 149 with comma separator, last without body = (single_row + ",") * 149 + single_row assert _is_grammar_accept_string(grammar, "[" + body + "]") mixed_row = "(" + "abc" * 43 + "a" + ")" body_mixed = (mixed_row + ",") * 149 + mixed_row assert _is_grammar_accept_string(grammar, "[" + body_mixed + "]") # Too few cells in a row short_row = "(" + "a" * 129 + ")" bad_body = (short_row + ",") * 149 + short_row assert not _is_grammar_accept_string(grammar, "[" + bad_body + "]") # Too few rows body_few = (single_row + ",") * 148 + single_row assert not _is_grammar_accept_string(grammar, "[" + body_few + "]") if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.2.3/tests/python/test_grammar_matcher_json.py000066400000000000000000000322631521764210300236170ustar00rootroot00000000000000"""This test uses the optimized JSON grammar provided by the grammar library.""" import sys import time from typing import List import pytest import torch from transformers import AutoTokenizer import xgrammar as xgr from xgrammar.testing import _get_masked_tokens_from_bitmask, _is_grammar_accept_string json_grammar = xgr.Grammar.builtin_json_grammar() json_input_accepted = [ '{"name": "John"}', '{ "name" : "John" }', "{}", "[]", '{"name": "Alice", "age": 30, "city": "New York"}', '{"name": "Mike", "hobbies": ["reading", "cycling", "hiking"]}', '{"name": "Emma", "address": {"street": "Maple Street", "city": "Boston"}}', '[{"name": "David"}, {"name": "Sophia"}]', ( '{"name": "William", "age": null, "married": true, "children": ["Liam", "Olivia"],' ' "hasPets": false}' ), ( '{"name": "Olivia", "contact": {"email": "olivia@example.com", "address": ' '{"city": "Chicago", "zipcode": "60601"}}}' ), ( '{"name": "Liam", "skills": ["Java", "Python"], "experience": ' '[{"company": "CompanyA", "years": 5}, {"company": "CompanyB", "years": 3}]}' ), ( '{"person": {"name": "Ethan", "age": 40}, "education": {"degree": "Masters", ' '"university": "XYZ University"}, "work": [{"company": "ABC Corp", "position": ' '"Manager"}, {"company": "DEF Corp", "position": "Senior Manager"}]}' ), ( '{"name": "Charlotte", "details": {"personal": {"age": 35, "hobbies": ["gardening", ' '"painting"]}, "professional": {"occupation": "Engineer", "skills": ' '["CAD", "Project Management"], "projects": [{"name": "Project A", ' '"status": "Completed"}, {"name": "Project B", "status": "In Progress"}]}}}' ), ] @pytest.mark.parametrize("json_input_accepted", json_input_accepted) def test_json_accept(json_input_accepted: str): assert _is_grammar_accept_string(json_grammar, json_input_accepted) json_input_refused = ( r'{ name: "John" }', r'{ "name": "John" } ', # trailing space is not accepted r'{ "name": "John", "age": 30, }', r'{ "name": "John", "address": { "street": "123 Main St", "city": "New York" }', r'{ "name": "John", "age": 30, "hobbies": ["reading", "traveling",], }', r'{ "name": "John", "age": 30.5.7 }', r'{ "name": "John, "age": 30, "hobbies": ["reading", "traveling"] }', ( r'{ "name": "John", "age": 30, "hobbies": ["reading", { "type": "outdoor", "list": ' r'["hiking", "swimming",]}] }' ), r'{ "name": "John", "age": 30, "status": "\P\J" }', ( r'{ "name": "John", "age": 30, "hobbies": ["reading", "traveling"], "address": ' r'{ "street": "123 Main St", "city": "New York", "coordinates": { "latitude": 40.7128, ' r'"longitude": -74.0060 }}}, "work": { "company": "Acme", "position": "developer" }}' ), ) @pytest.mark.parametrize("json_input_refused", json_input_refused) def test_json_refuse(json_input_refused: str): assert not _is_grammar_accept_string(json_grammar, json_input_refused) json_input_pressure = ( # Extra long string: 1k chars ( '["Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer nec odio. Praesent ' "libero. Sed cursus ante dapibus diam. Sed nisi. Nulla quis sem at nibh elementum " "imperdiet. Duis sagittis ipsum. Praesent mauris. Fusce nec tellus sed augue semper " "porta. Mauris massa. Vestibulum lacinia arcu eget nulla. Class aptent taciti sociosqu " "ad litora torquent per conubia nostra, per inceptos himenaeos. Curabitur sodales ligula " "in libero. Sed dignissim lacinia nunc. Curabitur tortor. Pellentesque nibh. Aenean quam. " "In scelerisque sem at dolor. Maecenas mattis. Sed convallis tristique sem. Proin ut " "ligula vel nunc egestas porttitor. Morbi lectus risus, iaculis vel, suscipit quis, " "luctus non, massa. Fusce ac turpis quis ligula lacinia aliquet. Mauris ipsum. Nulla " "metus metus, ullamcorper vel, tincidunt sed, euismod in, nibh. Quisque volutpat " "condimentum velit. Class aptent taciti sociosqu ad litora torquent per conubia nostra, " "per inceptos himenaeos. Nam nec ante. Sed lacinia, urna non tincidunt mattis, tortor " "neque adipiscing diam, a cursus ipsum ante quis turpis. Nulla facilisi. Ut fringilla. " "Suspendisse potenti. Nunc feugiat mi a tellus consequat imperdiet. Vestibulum sapien. " "Proin quam. Etiam ultrices. Suspendisse in justo eu magna luctus suscipit. Sed lectus. " "Integer euismod lacus luctus magna. Quisque cursus, metus vitae pharetra auctor, sem " 'massa mattis sem, at interdum magna augue eget diam."]' ), # long and complex json: 3k chars ( r"""{ "web-app": { "servlet": [ { "servlet-name": "cofaxCDS", "servlet-class": "org.cofax.cds.CDSServlet", "init-param": { "configGlossary:installationAt": "Philadelphia, PA", "configGlossary:adminEmail": "ksm@pobox.com", "configGlossary:poweredBy": "Cofax", "configGlossary:poweredByIcon": "/images/cofax.gif", "configGlossary:staticPath": "/content/static", "templateProcessorClass": "org.cofax.WysiwygTemplate", "templateLoaderClass": "org.cofax.FilesTemplateLoader", "templatePath": "templates", "templateOverridePath": "", "defaultListTemplate": "listTemplate.htm", "defaultFileTemplate": "articleTemplate.htm", "useJSP": false, "jspListTemplate": "listTemplate.jsp", "jspFileTemplate": "articleTemplate.jsp", "cachePackageTagsTrack": 200, "cachePackageTagsStore": 200, "cachePackageTagsRefresh": 60, "cacheTemplatesTrack": 100, "cacheTemplatesStore": 50, "cacheTemplatesRefresh": 15, "cachePagesTrack": 200, "cachePagesStore": 100, "cachePagesRefresh": 10, "cachePagesDirtyRead": 10, "searchEngineListTemplate": "forSearchEnginesList.htm", "searchEngineFileTemplate": "forSearchEngines.htm", "searchEngineRobotsDb": "WEB-INF/robots.db", "useDataStore": true, "dataStoreClass": "org.cofax.SqlDataStore", "redirectionClass": "org.cofax.SqlRedirection", "dataStoreName": "cofax", "dataStoreDriver": "com.microsoft.jdbc.sqlserver.SQLServerDriver", "dataStoreUrl": "jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon", "dataStoreUser": "sa", "dataStorePassword": "dataStoreTestQuery", "dataStoreTestQuery": "SET NOCOUNT ON;select test='test';", "dataStoreLogFile": "/usr/local/tomcat/logs/datastore.log", "dataStoreInitConns": 10, "dataStoreMaxConns": 100, "dataStoreConnUsageLimit": 100, "dataStoreLogLevel": "debug", "maxUrlLength": 500 } }, { "servlet-name": "cofaxEmail", "servlet-class": "org.cofax.cds.EmailServlet", "init-param": { "mailHost": "mail1", "mailHostOverride": "mail2" } }, { "servlet-name": "cofaxAdmin", "servlet-class": "org.cofax.cds.AdminServlet" }, { "servlet-name": "fileServlet", "servlet-class": "org.cofax.cds.FileServlet" }, { "servlet-name": "cofaxTools", "servlet-class": "org.cofax.cms.CofaxToolsServlet", "init-param": { "templatePath": "toolstemplates/", "log": 1, "logLocation": "/usr/local/tomcat/logs/CofaxTools.log", "logMaxSize": "", "dataLog": 1, "dataLogLocation": "/usr/local/tomcat/logs/dataLog.log", "dataLogMaxSize": "", "removePageCache": "/content/admin/remove?cache=pages&id=", "removeTemplateCache": "/content/admin/remove?cache=templates&id=", "fileTransferFolder": "/usr/local/tomcat/webapps/content/fileTransferFolder", "lookInContext": 1, "adminGroupID": 4, "betaServer": true } } ], "servlet-mapping": { "cofaxCDS": "/", "cofaxEmail": "/cofaxutil/aemail/*", "cofaxAdmin": "/admin/*", "fileServlet": "/static/*", "cofaxTools": "/tools/*" }, "taglib": { "taglib-uri": "cofax.tld", "taglib-location": "/WEB-INF/tlds/cofax.tld" } } }""" ), ) @pytest.mark.parametrize("json_input_pressure", json_input_pressure) def test_json_pressure(json_input_pressure: str): assert _is_grammar_accept_string(json_grammar, json_input_pressure, print_time=True) tokenizer_path__input_str__expected_rejected_sizes = [ ( # short test "meta-llama/Llama-2-7b-chat-hf", '{"id": 1,"name": "Example"}', [ # fmt: off 31989, 31912, 270, 270, 270, 31973, 31846, 31846, 31948, 31915, 270, 270, 270, 270, 270, 31973, 31846, 31846, 263, 263, 263, 263, 263, 263, 263, 263, 31974, 31999, # fmt: on ], ), ( # long test "meta-llama/Llama-2-7b-chat-hf", """{ "id": 1, "na": "ex", "ac": true, "t": ["t1", "t2"], "ne": {"lv2": {"val": "dp"}, "arr": [1, 2, 3]}, "res": "res" }""", [ # fmt: off 31989, 31912, 31912, 270, 270, 270, 31973, 31846, 31846, 31948, 31915, 31915, 270, 270, 270, 31973, 31846, 31846, 263, 263, 263, 31974, 31915, 31915, 270, 270, 270, 31973, 31846, 31846, 31997, 31997, 31998, 31974, 31915, 31915, 270, 270, 31973, 31846, 31846, 31840, 262, 262, 262, 31969, 31846, 31846, 262, 262, 262, 31969, 31974, 31915, 31915, 270, 270, 270, 31973, 31846, 31846, 31908, 270, 270, 270, 270, 31973, 31846, 31846, 31906, 270, 270, 270, 270, 31973, 31846, 31846, 262, 262, 262, 31968, 31970, 31915, 31915, 270, 270, 270, 270, 31973, 31846, 31846, 31840, 31943, 31846, 31846, 31943, 31846, 31846, 31943, 31970, 31974, 31915, 31915, 270, 270, 270, 270, 31973, 31846, 31846, 263, 263, 263, 263, 31974, 31974, 31999, # fmt: on ], ), ( # test for llama 3 "meta-llama/Meta-Llama-3-8B-Instruct", '{"id": 1,"name": "Example哈哈"}', [ # fmt: off 128235, 127497, 4744, 4744, 4744, 127849, 126399, 126399, 126760, 127499, 4744, 4744, 4744, 4744, 4744, 127849, 126399, 126399, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 128066, 128111, 4694, 128066, 128111, 4694, 127873, 128255, # fmt: on ], ), ] @pytest.mark.hf_token_required @pytest.mark.parametrize( "tokenizer_path, input_str, expected_rejected_sizes", tokenizer_path__input_str__expected_rejected_sizes, ) def test_fill_next_token_bitmask( tokenizer_path: str, input_str: str, expected_rejected_sizes: List[int] ): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) time_start = time.monotonic_ns() matcher = xgr.GrammarMatcher(compiler.compile_builtin_json_grammar()) time_end = time.monotonic_ns() print(f"Time to init GrammarMatcher: {(time_end - time_start) / 1e3} us") token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) device = "cuda" if torch.cuda.is_available() else "cpu" logits_gpu = torch.zeros(1, tokenizer_info.vocab_size, dtype=torch.float32, device=device) input_bytes = input_str.encode("utf-8") for i, c in enumerate(input_bytes): # 1. fill_next_token_bitmask time_start = time.monotonic_ns() assert matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time to fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") # 2. Correctness verification rejected_token_ids = _get_masked_tokens_from_bitmask( token_bitmask, tokenizer_info.vocab_size ) assert len(rejected_token_ids) == expected_rejected_sizes[i] # 3. apply_token_bitmask_inplace if torch.cuda.is_available(): torch.cuda.synchronize() time_start = time.monotonic_ns() xgr.apply_token_bitmask_inplace(logits_gpu, token_bitmask.to(device)) if torch.cuda.is_available(): torch.cuda.synchronize() time_end = time.monotonic_ns() print(f"Time to apply_token_bitmask_inplace: {(time_end - time_start) / 1e3} us") # 4. accept_string print("Accepting char:", bytes([c])) time_start = time.monotonic_ns() assert matcher.accept_string(bytes([c])) time_end = time.monotonic_ns() print(f"Time to accept_token: {(time_end - time_start) / 1e3} us") # 5. Final correctness verification matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size) assert len(rejected_token_ids) == expected_rejected_sizes[-1] if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.2.3/tests/python/test_grammar_matcher_json_schema.py000066400000000000000000001246121521764210300251370ustar00rootroot00000000000000import json import sys import time from typing import Dict, List, Tuple import pytest from pydantic import BaseModel, Field from transformers import AutoConfig, AutoTokenizer import xgrammar as xgr from xgrammar.testing import ( _get_masked_tokens_from_bitmask, _get_matcher_from_grammar_and_tokenizer_info, _is_grammar_accept_string, ) class MainModel(BaseModel): integer_field: int number_field: float boolean_field: bool any_array_field: List array_field: List[str] tuple_field: Tuple[str, int, List[str]] object_field: Dict[str, int] nested_object_field: Dict[str, Dict[str, int]] instance = MainModel( integer_field=42, number_field=3.14e5, boolean_field=True, any_array_field=[3.14, "foo", None, True], array_field=["foo", "bar"], tuple_field=("foo", 42, ["bar", "baz"]), object_field={"foo": 42, "bar": 43}, nested_object_field={"foo": {"bar": 42}}, ) instance_str = instance.model_dump_json(indent=2, round_trip=True) @pytest.mark.hf_token_required def test_json_schema_debug_accept_string(): grammar = xgr.Grammar.from_json_schema(MainModel, indent=2) instance = MainModel( integer_field=42, number_field=3.14e5, boolean_field=True, any_array_field=[3.14, "foo", None, True], array_field=["foo", "bar"], tuple_field=("foo", 42, ["bar", "baz"]), object_field={"foo": 42, "bar": 43}, nested_object_field={"foo": {"bar": 42}}, ) instance_str = instance.model_dump_json(indent=2, round_trip=True) tokenizer_path = "meta-llama/Llama-2-7b-chat-hf" tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) matcher = _get_matcher_from_grammar_and_tokenizer_info(grammar, tokenizer_info) for c in instance_str: assert matcher.accept_string(c) assert matcher.accept_token(2) assert matcher.is_terminated() def test_json_schema_find_jump_forward_string(): grammar = xgr.Grammar.from_json_schema(MainModel, indent=2) matcher = _get_matcher_from_grammar_and_tokenizer_info(grammar, xgr.TokenizerInfo([])) for i, c in enumerate(instance_str): jump_forward_str = matcher.find_jump_forward_string() assert instance_str[i : i + len(jump_forward_str)] == jump_forward_str assert matcher.accept_string(c) assert matcher.find_jump_forward_string() == "" tokenizer_path = ["meta-llama/Llama-2-7b-chat-hf", "meta-llama/Meta-Llama-3-8B-Instruct"] @pytest.mark.hf_token_required @pytest.mark.parametrize("tokenizer_path", tokenizer_path) def test_fill_next_token_bitmask(tokenizer_path: str): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) time_start = time.monotonic_ns() compiled_grammar = compiler.compile_json_schema(MainModel, indent=2) matcher = xgr.GrammarMatcher(compiled_grammar) time_end = time.monotonic_ns() print(f"Time to init GrammarMatcher: {(time_end - time_start) / 1e3} us") token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) input_bytes = instance_str.encode("utf-8") for _, c in enumerate(input_bytes): # 1. fill_next_token_bitmask time_start = time.monotonic_ns() matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time to fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") # 2. accept_string print("Accepting char:", bytes([c])) time_start = time.monotonic_ns() assert matcher.accept_string(bytes([c])) time_end = time.monotonic_ns() print(f"Time to accept_token: {(time_end - time_start) / 1e3} us") # 3. Final correctness verification matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size) assert tokenizer.eos_token_id not in rejected_token_ids class RangeSchema(BaseModel): value: int = Field(ge=1, le=100) class ExtendedRangeSchema(BaseModel): value: int = Field(ge=-128, le=256) class NegativeRangeSchema(BaseModel): value: int = Field(ge=-1000, le=-1) class LargeRangeSchema(BaseModel): value: int = Field(ge=-99999, le=99999) class LargeRangeSchemaStartZero(BaseModel): value: int = Field(ge=0, le=20_000_000_000) class FloatRangeSchema(BaseModel): value: float = Field(ge=0.0, le=1.0) class NegativeFloatRangeSchema(BaseModel): value: float = Field(ge=-10.0, le=-0.1) class ComplexFloatRangeSchema(BaseModel): value: float = Field(ge=-12345.12345, le=56789.56789) class LargeFloatRangeSchema(BaseModel): value: float = Field(ge=-1000.0, le=1000.0) class MultipleBoundariesSchema(BaseModel): small_value: int = Field(ge=-10, le=10) medium_value: int = Field(ge=-100, le=100) large_value: int = Field(ge=-1000, le=1000) class MixedTypeRangeSchema(BaseModel): int_value: int = Field(ge=-100, le=100) float_value: float = Field(ge=-10.0, le=10.0) class VeryLargeFloatRangeSchema(BaseModel): value: float = Field(ge=-20_000_000_000.123123, le=20_000_000_000.456789) class ExceedsInt64MaxSchema(BaseModel): value: int = Field(ge=0, le=18446744073709551615) class ExceedsInt64MinSchema(BaseModel): value: int = Field(ge=-9223372036854775809, le=100) class ExceedsInt64RangeSchema(BaseModel): value: int = Field(ge=-18446744073709551616, le=18446744073709551616) class ValidInt64MaxSchema(BaseModel): value: int = Field(ge=0, le=9223372036854775807) class ValidInt64MinSchema(BaseModel): value: int = Field(ge=-9223372036854775808, le=0) class ValidLargeIntSchema(BaseModel): value: int = Field(ge=0, le=1000000000000000000) @pytest.mark.parametrize("tokenizer_path", tokenizer_path) @pytest.mark.parametrize( "schema_class,test_value", [ # Integer test cases (RangeSchema, 42), (ExtendedRangeSchema, -128), (ExtendedRangeSchema, 0), (ExtendedRangeSchema, 256), (ExtendedRangeSchema, 14), (NegativeRangeSchema, -1000), (NegativeRangeSchema, -500), (NegativeRangeSchema, -1), (LargeRangeSchema, -99999), (LargeRangeSchema, -5678), (LargeRangeSchema, 0), (LargeRangeSchema, 5678), (LargeRangeSchema, 99999), (LargeRangeSchemaStartZero, 20000000000), (LargeRangeSchemaStartZero, 0), (LargeRangeSchemaStartZero, 10000000000), (LargeRangeSchemaStartZero, 19999999999), # Float test cases (FloatRangeSchema, 0.0), (FloatRangeSchema, 0.5), (FloatRangeSchema, 1.0), (NegativeFloatRangeSchema, -10.0), (NegativeFloatRangeSchema, -5.5), (NegativeFloatRangeSchema, -0.1), (LargeFloatRangeSchema, -1000.0), (LargeFloatRangeSchema, -500.5), (LargeFloatRangeSchema, 0.0), (LargeFloatRangeSchema, 500.5), (LargeFloatRangeSchema, 1000.0), (ComplexFloatRangeSchema, (-1234.1234)), (ComplexFloatRangeSchema, (0)), (ComplexFloatRangeSchema, (5671.123456)), (VeryLargeFloatRangeSchema, (20_000_000_000.456788)), (VeryLargeFloatRangeSchema, (-19_999_999_999.456789)), # Signed 64-bit boundary test cases (should succeed) (ValidInt64MaxSchema, 9223372036854775807), (ValidInt64MaxSchema, 1000), (ValidInt64MinSchema, -9223372036854775808), (ValidInt64MinSchema, -1000), (ValidLargeIntSchema, 1000000000000000000), (ValidLargeIntSchema, 1000), ], ) @pytest.mark.hf_token_required def test_fill_next_token_bitmask_intfloat_range(tokenizer_path: str, schema_class, test_value): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) instance = schema_class(value=test_value) instance_str = instance.model_dump_json() print(f"Testing {schema_class.__name__} with value {test_value}") time_start = time.monotonic_ns() compiled_grammar = compiler.compile_json_schema(schema_class) matcher = xgr.GrammarMatcher(compiled_grammar) time_end = time.monotonic_ns() print(f"Time to init GrammarMatcher: {(time_end - time_start) / 1e3} us") token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) input_bytes = instance_str.encode("utf-8") for c in input_bytes: time_start = time.monotonic_ns() matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time to fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") assert matcher.accept_string(bytes([c])) matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size) assert tokenizer.eos_token_id not in rejected_token_ids @pytest.mark.parametrize("tokenizer_path", tokenizer_path) @pytest.mark.parametrize( "schema_class,should_fail,error_pattern", [ (ExceedsInt64MaxSchema, True, "exceeds"), (ExceedsInt64MinSchema, True, "exceeds"), (ExceedsInt64RangeSchema, True, "exceeds"), ], ) @pytest.mark.hf_token_required def test_64bit_limit_validation( tokenizer_path: str, schema_class, should_fail: bool, error_pattern: str ): """Test that schemas exceeding signed 64-bit integer limits are properly rejected""" tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) if should_fail: with pytest.raises((ValueError, OverflowError, RuntimeError)) as exc_info: compiler.compile_json_schema(schema_class) assert error_pattern.lower() in str(exc_info.value).lower() @pytest.mark.parametrize("tokenizer_path", tokenizer_path) @pytest.mark.parametrize( "boundary_value,schema_class", [ (9223372036854775807, ValidInt64MaxSchema), (-9223372036854775808, ValidInt64MinSchema), (1000000000000000000, ValidLargeIntSchema), ], ) @pytest.mark.hf_token_required def test_signed_64bit_boundary_values_work(tokenizer_path: str, boundary_value: int, schema_class): """Test that signed 64-bit boundary values work correctly""" tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) try: compiled_grammar = compiler.compile_json_schema(schema_class) matcher = xgr.GrammarMatcher(compiled_grammar) test_value = min(abs(boundary_value), 1000) if boundary_value != 0 else 1000 if boundary_value < 0: test_value = -test_value test_instance = schema_class(value=test_value) instance_str = test_instance.model_dump_json() token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) for c in instance_str.encode("utf-8"): matcher.fill_next_token_bitmask(token_bitmask) assert matcher.accept_string(bytes([c])) except Exception as e: pytest.fail(f"Signed 64-bit boundary value {boundary_value} unexpectedly failed: {e}") @pytest.mark.hf_token_required @pytest.mark.parametrize("tokenizer_path", tokenizer_path) def test_mixed_type_range_schema(tokenizer_path: str): """Test the MixedTypeRangeSchema with both integer and float fields""" tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) test_instances = [ MixedTypeRangeSchema(int_value=-100, float_value=-10.0), MixedTypeRangeSchema(int_value=100, float_value=10.0), MixedTypeRangeSchema(int_value=0, float_value=0.0), MixedTypeRangeSchema(int_value=-50, float_value=5.5), ] for instance in test_instances: instance_str = instance.model_dump_json() print(f"Testing MixedTypeRangeSchema with values: {instance}") time_start = time.monotonic_ns() compiled_grammar = compiler.compile_json_schema(MixedTypeRangeSchema) matcher = xgr.GrammarMatcher(compiled_grammar) time_end = time.monotonic_ns() print(f"Time to init GrammarMatcher: {(time_end - time_start) / 1e3} us") token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) input_bytes = instance_str.encode("utf-8") for c in input_bytes: time_start = time.monotonic_ns() matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time to fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") assert matcher.accept_string(bytes([c])) matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask( token_bitmask, tokenizer_info.vocab_size ) assert tokenizer.eos_token_id not in rejected_token_ids @pytest.mark.hf_token_required @pytest.mark.parametrize("tokenizer_path", tokenizer_path) def test_multiple_boundaries_schema(tokenizer_path: str): """Test the complex MultipleBoundariesSchema with multiple integer fields""" tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) test_instances = [ MultipleBoundariesSchema( small_value=-10, medium_value=-100, large_value=-1000 ), # All lower bounds MultipleBoundariesSchema( small_value=10, medium_value=100, large_value=1000 ), # All upper bounds MultipleBoundariesSchema(small_value=0, medium_value=0, large_value=0), MultipleBoundariesSchema(small_value=-5, medium_value=50, large_value=-500), ] for instance in test_instances: instance_str = instance.model_dump_json() print(f"Testing MultipleBoundariesSchema with values: {instance}") time_start = time.monotonic_ns() compiled_grammar = compiler.compile_json_schema(MultipleBoundariesSchema) matcher = xgr.GrammarMatcher(compiled_grammar) time_end = time.monotonic_ns() print(f"Time to init GrammarMatcher: {(time_end - time_start) / 1e3} us") token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) input_bytes = instance_str.encode("utf-8") for c in input_bytes: time_start = time.monotonic_ns() matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time to fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") assert matcher.accept_string(bytes([c])) matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask( token_bitmask, tokenizer_info.vocab_size ) assert tokenizer.eos_token_id not in rejected_token_ids string_format_instances = [ (r"long.email-address-with-hyphens@and.subdomains.example.com", "email"), (r'"very.(),:;<>[]\".VERY.\"very@\\ \"very\".unusual"@strange.example.com', "email"), (r"128.255.000.222", "ipv4"), (r"2001:db8:3:4::192.0.2.33", "ipv6"), (r"P1Y23M456DT9H87M654S", "duration"), (r"2025-01-01T12:34:56.7+08:09", "date-time"), (r"123--abc.efgh---789-xyz.rst-uvw", "hostname"), (r"01234567-89AB-CDEF-abcd-ef0123456789", "uuid"), ( r"http://azAZ09-._~%Ff!$&'()*+,;=:@xyz:987/-/./+/*?aA0-._~%Ff!$&'()@#zZ9-._~%Aa!$&,;=:", "uri", ), ] # not frequently used string_format_instances_skipped = [ ( r"//azAZ09-._~%Ff!$&'()*+,;=:@xyz:987/-/./+/*?aA0-._~%Ff!$&'()@#zZ9-._~%Aa!$&,;=:", "uri-reference", ), (r"!#$&()*+,-./{+abc}{#def}{.ghi}{/jkl}{;mno:2468}", "uri-template"), (r"/a/bc/def/ghij/~0~1//", "json-pointer"), (r"1234/a/bc/def/ghij/~0~1//", "relative-json-pointer"), ] @pytest.mark.hf_token_required @pytest.mark.parametrize("value, format", string_format_instances) def test_mask_generation_format(value: str, format: str): class MainModel(BaseModel): name: str = Field(json_schema_extra={"format": format}) instance = json.dumps(MainModel(name=value).model_dump(mode="json")) tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3.1-8B-Instruct") tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) grammar_compiler = xgr.GrammarCompiler(tokenizer_info, cache_enabled=False) time_start = time.monotonic_ns() compiled_grammar = grammar_compiler.compile_json_schema(MainModel) time_end = time.monotonic_ns() print(f"Time for preprocessing: {(time_end - time_start) / 1e3} us") matcher = xgr.GrammarMatcher(compiled_grammar) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) for c in instance.encode("utf-8"): time_start = time.monotonic_ns() matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() delta_us = (time_end - time_start) / 1e3 print(f"Time for fill_next_token_bitmask: {delta_us} us before accepting char {bytes([c])}") accepted = matcher.accept_string(bytes([c])) assert accepted time_start = time.monotonic_ns() matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time for fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") assert matcher.accept_token(tokenizer.eos_token_id) assert matcher.is_terminated() @pytest.mark.hf_token_required def test_implicit_left_recursion_schema(): model_name = "meta-llama/Llama-3.2-1B-Instruct" tokenizer = AutoTokenizer.from_pretrained(model_name) config = AutoConfig.from_pretrained(model_name) json_schema = { "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "properties": { "url": { "type": "string", "pattern": "^(https?://)?([\\da-z\\.-]+)\\.([a-z\\.]{2,6})([/\\w \\.-]*)*/?", } }, } tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=config.vocab_size) grammar_compiler = xgr.GrammarCompiler(tokenizer_info) _ = grammar_compiler.compile_json_schema(schema=json.dumps(json_schema)) @pytest.mark.hf_token_required def test_regression_accept_invalid_token(): tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-235B-A22B-Instruct-2507-FP8") vocab_size = 151936 tokenizer_info = xgr.TokenizerInfo.from_huggingface( tokenizer, vocab_size=vocab_size, stop_token_ids=[tokenizer.eos_token_id] ) grammar_compiler = xgr.GrammarCompiler(tokenizer_info=tokenizer_info) ctx = grammar_compiler.compile_json_schema( schema=""" {"type": "object", "properties": {"value": {"type": ["string", "null"], "maxLength": 10}, "nested": {"type": "object", "properties": {"value": {"type": ["string", "null"]}, "nested_nested": {"type": "array", "items": {"type": ["string", "null"]}}}, "required": ["value", "nested_nested"], "maxItems": 10, "minItems": 1}}, "required": ["value", "nested"], "additionalProperties": false}""" ) matcher = xgr.GrammarMatcher(ctx, max_rollback_tokens=200, override_stop_tokens=None) token_bitmask = xgr.allocate_token_bitmask(vocab_size=vocab_size, batch_size=7) token_bitmask.fill_(0) for i, token in enumerate([4913, 957, 788, 330, 1072, 67212, 788]): if i == 0: accepted = True else: parent_pos = i - 1 curr_token_id = token parent_bitmask = token_bitmask[parent_pos] # 32 boolean bitmask values are packed into 32-bit integers accepted = (parent_bitmask[curr_token_id // 32] & (1 << (curr_token_id % 32))) != 0 assert matcher.accept_token(token) == accepted matcher.fill_next_token_bitmask(token_bitmask, i) @pytest.mark.hf_token_required def test_regression_accept_kimi_tokenizer_token(): config = AutoConfig.from_pretrained("moonshotai/Kimi-K2-Thinking", trust_remote_code=True) tokenizer = AutoTokenizer.from_pretrained("moonshotai/Kimi-K2-Thinking", trust_remote_code=True) vocab_size = config.vocab_size ids = tokenizer.encode( r'{"command": "find ./ -name *.txt ", "security_risk": "LOW"}', add_special_tokens=True ) tokens = tokenizer.convert_ids_to_tokens(ids) tokenizer_info = xgr.TokenizerInfo.from_huggingface( tokenizer, vocab_size=vocab_size, stop_token_ids=[tokenizer.eos_token_id] ) grammar_compiler = xgr.GrammarCompiler(tokenizer_info=tokenizer_info) schema = { "type": "object", "properties": { "command": {"type": "string"}, "security_risk": {"type": "string", "enum": ["LOW", "MEDIUM", "HIGH"]}, }, "required": ["command"], } ctx = grammar_compiler.compile_json_schema(schema=json.dumps(schema)) matcher = xgr.GrammarMatcher(ctx, max_rollback_tokens=200, override_stop_tokens=None) for i, token in zip(ids, tokens): assert matcher.accept_token(i) matcher.accept_token(tokenizer.eos_token_id) # accept EOS assert matcher.is_terminated() def test_regression_empty_property_key_regex(): schema = { "type": "object", "properties": { "_links": { "type": "object", "patternProperties": { "": {"type": "object", "properties": {"href": {"type": "string"}}} }, } }, } _ = xgr.Grammar.from_json_schema(schema) assert _ is not None def test_json_schema_number_without_constraint(): schema = {"type": "object", "properties": {"value": {"type": "number"}}, "required": ["value"]} grammar = xgr.Grammar.from_json_schema(schema) assert _is_grammar_accept_string(grammar, '{"value": -0.5}') assert _is_grammar_accept_string(grammar, '{"value": -1.5}') assert _is_grammar_accept_string(grammar, '{"value": 0}') assert _is_grammar_accept_string(grammar, '{"value": 1234567890}') assert _is_grammar_accept_string(grammar, '{"value": 3.14159}') assert _is_grammar_accept_string(grammar, '{"value": 1e10}') assert _is_grammar_accept_string(grammar, '{"value": -2.5E-3}') assert _is_grammar_accept_string(grammar, '{"value": 0.0}') assert _is_grammar_accept_string(grammar, '{"value": -0.0}') assert not _is_grammar_accept_string(grammar, '{"value": "abc"}') @pytest.mark.hf_token_required def test_rule_level_cache_cross_grammar(): """ This test ensures the result after applying the rule-level cache is consistent with the previous version (without rule-level cache). """ # fmt: off rejected_a = [128251, 127885, 127885, 127885, 127885, 127885, 127885, 128254, 128255, 128247, 127875, 127760, 127760, 91779, 91770, 91770, 91770, 91770, 91770, 91770, 91770, 91770, 91770, 91770, 127878, 127885, 127885, 127885, 127885, 127885, 127885, 128250, 128253, 128251, 128252, 128253, 128254, 128255, 128246, 127876, 127877, 127877, 127884, 127884, 127884, 127884, 127884, 127884, 127884, 127884, 127884, 127884, 128251, 128253, 128252, 128253, 128252, 128253, 128254, 128255, 128244, 127867, 127510, 127510, 3501, 3486, 3486, 3486, 3486, 3486, 127856, 127884, 127884, 127884, 127884, 127884, 127884, 127884, 127884, 127884, 127884, 128244, 128252, 128255, 128248, 127878, 126888, 126888, 126746, 126746, 127884, 127884, 127884, 127884, 127884, 127884, 127884, 127884, 127884, 127884, 128246, 128252, 128254, 128254, 128253, 128254, 128255, 128246, 127876, 127877, 127877, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 128252, 128252, 128253, 128254, 128255, 128244, 127873, 127710, 127710, 89619, 83629, 83629, 83629, 83629, 83629, 98988, 91326, 91326, 91326, 91326, 91326, 91326, 91326, 98988, 97216, 91326, 91326, 127847, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 128252, 128252, 128253, 128254, 128255, 128243, 127867, 127499, 127499, 4756, 4756, 4756, 4756, 4756, 4756, 4756, 4756, 4756, 4756, 4756, 4756, 4756, 4756, 4756, 4756, 4756, 127857, 127857, 127857, 127857, 127857, 127857, 127857, 127857, 127857, 127857, 127856, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 128251, 128254, 128252, 128252, 128253, 128254, 128255, 128246, 127876, 127877, 127877, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 128251, 128251, 128253, 128254, 128253, 128254, 128255, 128241, 127865, 127490, 127490, 4702, 4702, 4702, 4702, 4702, 4702, 127878, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 128252, 128253, 128254, 128255, 128241, 127865, 127489, 127489, 4694, 4694, 4694, 4694, 4694, 4694, 127855, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 128248, 128252, 128252, 128254, 128254, 128255, 128241, 127865, 127489, 127489, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 127855, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 128253, 128254, 128255, 128243, 127867, 127500, 127500, 4761, 4761, 4761, 4761, 4761, 4761, 4761, 4761, 4761, 127865, 127865, 127865, 127865, 127865, 127865, 127865, 127865, 127865, 127865, 127866, 127866, 127866, 127866, 127866, 127866, 127878, 127886, 127886, 127886, 127886, 127886, 127886, 128245, 128250, 128253, 128251, 128251, 128251, 128251, 128252, 128253, 128254, 128255, 128246, 127875, 127866, 127866, 127863, 127863, 127863, 127863, 127863, 127863, 127863, 127863, 127863, 127863, 128241, 128251, 128254, 128253, 128254, 128253, 128254, 128255, 128241, 127865, 127488, 127488, 4692, 4692, 4692, 127856, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 128246, 128253, 128253, 128254, 128252, 128253, 128254, 128255, 128241, 127865, 127488, 127488, 4692, 4692, 4692, 4692, 4692, 4692, 4692, 4692, 4692, 4692, 4692, 127856, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 128251, 128253, 128254, 128254, 128250, 128251, 128253, 128253, 128254, 128255, 128248, 127874, 127867, 127867, 128254, 128254, 128255, 127866, 127866, 127866, 127866, 127866, 127866, 127878, 127886, 127886, 127886, 127886, 127886, 127886, 128251, 128252, 128254, 128252, 128252, 128253, 128254, 128255, 128246, 127876, 127877, 127877, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 128250, 128250, 128252, 128252, 128253, 128254, 128255, 128253, 128254, 128255, 128247, 127876, 127886, 127886, 127146, 127146, 128146, 128246, 128255, 128242, 128253, 128255, 128221, 128247, 128255, 128229, 128246, 128255, 128190, 128246, 128255, 128188, 128246, 128252, 128243, 127862, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 128247, 128254, 128251, 128252, 128253, 128254, 128255, 128253, 128254, 128255, 128247, 127876, 127886, 127886, 127146, 127146, 128146, 128246, 128255, 128242, 128247, 128255, 128221, 128247, 128255, 128229, 128246, 128255, 128190, 128246, 128255, 128188, 128246, 128252, 128243, 127862, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 128252, 128253, 128254, 128255, 128246, 127875, 127869, 127869, 127478, 4684, 4684, 4684, 4684, 4684, 4684, 4684, 4684, 4684, 4684, 127863, 127490, 127490, 4684, 4684, 4684, 4684, 4684, 4684, 4684, 4684, 127863, 127872, 127872, 127872, 127872, 127872, 127872, 127886, 127886, 127886, 127886, 127886, 127886, 128255] rejected_b = [128251, 127885, 127885, 127885, 127885, 127885, 127885, 128254, 128255, 128247, 127875, 127760, 127760, 91779, 91770, 91770, 91770, 91770, 91770, 91770, 91770, 91770, 91770, 91770, 91770, 127878, 127885, 127885, 127885, 127885, 127885, 127885, 128250, 128253, 128251, 128252, 128253, 128254, 128255, 128246, 127876, 127877, 127877, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 128251, 128252, 128253, 128254, 128255, 128244, 127867, 127510, 127510, 3501, 3486, 3486, 3486, 3486, 3486, 3486, 3486, 3486, 3486, 3486, 3486, 127856, 127883, 127883, 127883, 127883, 127883, 127883, 127883, 127883, 127883, 127883, 128242, 128252, 128254, 128254, 128252, 128253, 128254, 128255, 128251, 128251, 128252, 128254, 128254, 128253, 128254, 128255, 128241, 127865, 127488, 127488, 4692, 4692, 4692, 4692, 4692, 4692, 4692, 127856, 127884, 127884, 127884, 127884, 127884, 127884, 127884, 127884, 127884, 127884, 128246, 128252, 128254, 128254, 128253, 128254, 128255, 128246, 127876, 127877, 127877, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 128252, 128252, 128253, 128254, 128255, 128244, 127873, 127710, 127710, 89619, 83629, 83629, 83629, 83629, 83629, 83629, 83629, 98988, 91326, 91326, 91326, 91326, 91326, 91326, 91326, 98988, 97216, 91326, 91326, 127847, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 128252, 128252, 128253, 128254, 128255, 128243, 127867, 127499, 127499, 4756, 4756, 4756, 4756, 4756, 4756, 4756, 4756, 4756, 4756, 4756, 4756, 4756, 4756, 4756, 4756, 127857, 127857, 127857, 127857, 127857, 127857, 127857, 127857, 127857, 127857, 127856, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 128251, 128254, 128252, 128252, 128253, 128254, 128255, 128246, 127876, 127877, 127877, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 128251, 128251, 128253, 128254, 128253, 128254, 128255, 128241, 127865, 127490, 127490, 4702, 4702, 4702, 4702, 4702, 4702, 127878, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 128252, 128253, 128254, 128255, 128241, 127865, 127489, 127489, 4694, 4694, 4694, 4694, 4694, 4694, 127855, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 128248, 128252, 128252, 128254, 128254, 128255, 128241, 127865, 127489, 127489, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 127855, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 128253, 128254, 128255, 128243, 127867, 127500, 127500, 4761, 4761, 4761, 4761, 4761, 4761, 4761, 4761, 4761, 127865, 127865, 127865, 127865, 127865, 127865, 127865, 127865, 127865, 127865, 127866, 127866, 127866, 127866, 127866, 127866, 127878, 127885, 127885, 127885, 127885, 127885, 127885, 128245, 128254, 128254, 128252, 128253, 128254, 128251, 128251, 128251, 128253, 128253, 128254, 128255, 128246, 127875, 127866, 127866, 127863, 127863, 127863, 127863, 127863, 127863, 127863, 127863, 127863, 127863, 128247, 128252, 128253, 128254, 128255, 128251, 128252, 128253, 128254, 128255, 128247, 127876, 127885, 127885, 128249, 128253, 128241, 127856, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 128251, 128252, 128253, 128253, 128252, 128253, 128254, 128255, 128251, 128252, 128253, 128254, 128255, 128248, 127878, 126889, 126889, 126756, 126756, 126756, 127866, 127866, 127866, 127866, 127866, 127878, 127886, 127886, 127886, 127886, 127886, 127886, 128251, 128252, 128254, 128252, 128252, 128253, 128254, 128255, 128246, 127876, 127877, 127877, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 127885, 128250, 128250, 128252, 128252, 128253, 128254, 128255, 128253, 128254, 128255, 128247, 127876, 127886, 127886, 127146, 127146, 128146, 128246, 128255, 128242, 128253, 128255, 128221, 128246, 128255, 128229, 128246, 128255, 128190, 128246, 128255, 128188, 128246, 128252, 128243, 127862, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 128247, 128254, 128251, 128252, 128253, 128254, 128255, 128253, 128254, 128255, 128247, 127876, 127886, 127886, 127146, 127146, 128146, 128246, 128255, 128242, 128247, 128255, 128221, 128247, 128255, 128229, 128246, 128255, 128190, 128246, 128255, 128188, 128246, 128252, 128243, 127862, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 127886, 128252, 128253, 128254, 128255, 128246, 127875, 127869, 127869, 127478, 4684, 4684, 4684, 4684, 4684, 4684, 4684, 4684, 127863, 127490, 127490, 4684, 4684, 4684, 4684, 4684, 4684, 4684, 4684, 4684, 4684, 127863, 127872, 127872, 127872, 127872, 127872, 127872, 127886, 127886, 127886, 127886, 127886, 127886, 128255] # fmt: on schema_a = { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "id": {"type": "string", "pattern": "^[a-zA-Z0-9_-]+$"}, "profile": { "type": "object", "properties": { "username": {"type": "string", "minLength": 1}, "age": {"type": "integer", "minimum": 0}, "contact": { "type": "object", "properties": { "email": {"type": "string", "format": "email"}, "phone": {"type": "string"}, }, "required": ["email"], "additionalProperties": False, }, "address": { "type": "object", "properties": { "country": {"type": "string"}, "city": {"type": "string"}, "street": {"type": "string"}, "zip": {"type": "string"}, }, "required": ["country", "city"], "additionalProperties": False, }, }, "required": ["username"], "additionalProperties": False, }, "preferences": { "type": "object", "properties": { "language": {"type": "string"}, "timezone": {"type": "string"}, "newsletter": {"type": "boolean"}, }, "additionalProperties": False, }, "metadata": { "type": "object", "properties": { "created_at": {"type": "string", "format": "date-time"}, "updated_at": {"type": "string", "format": "date-time"}, "tags": {"type": "array", "items": {"type": "string"}}, }, "required": ["created_at"], "additionalProperties": False, }, }, "required": ["id", "profile", "metadata"], "additionalProperties": False, } schema_b = { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "id": {"type": "string", "pattern": "^[a-zA-Z0-9_-]+$"}, "profile": { "type": "object", "properties": { "model": {"type": "string", "minLength": 1}, "firmware_version": {"type": "string"}, "contact": { "type": "object", "properties": { "email": {"type": "string", "format": "email"}, "phone": {"type": "string"}, }, "required": ["email"], "additionalProperties": False, }, "address": { "type": "object", "properties": { "country": {"type": "string"}, "city": {"type": "string"}, "street": {"type": "string"}, "zip": {"type": "string"}, }, "required": ["country", "city"], "additionalProperties": False, }, }, "required": ["model"], "additionalProperties": False, }, "configuration": { "type": "object", "properties": { "power_mode": {"type": "string", "enum": ["on", "off", "sleep"]}, "sampling_rate": {"type": "integer", "minimum": 1}, }, "additionalProperties": False, }, "metadata": { "type": "object", "properties": { "created_at": {"type": "string", "format": "date-time"}, "updated_at": {"type": "string", "format": "date-time"}, "tags": {"type": "array", "items": {"type": "string"}}, }, "required": ["created_at"], "additionalProperties": False, }, }, "required": ["id", "profile", "metadata"], "additionalProperties": False, } string_a = r"""{ "id": "user_12345", "profile": { "username": "alice", "age": 28, "contact": { "email": "alice@example.com", "phone": "+81-90-1234-5678" }, "address": { "country": "Japan", "city": "Tokyo", "street": "Chiyoda 1-1", "zip": "100-0001" } }, "preferences": { "language": "ja", "timezone": "Asia/Tokyo", "newsletter": true }, "metadata": { "created_at": "2025-12-01T10:15:30Z", "updated_at": "2026-01-02T08:20:00Z", "tags": ["beta_user", "premium"] } }""" string_b = r"""{ "id": "device_A9X3", "profile": { "model": "SensorPro-X", "firmware_version": "v2.3.1", "contact": { "email": "support@example.com", "phone": "+1-800-555-0199" }, "address": { "country": "Japan", "city": "Osaka", "street": "Namba 2-3-4", "zip": "542-0076" } }, "configuration": { "power_mode": "on", "sampling_rate": 100 }, "metadata": { "created_at": "2025-11-20T03:45:10Z", "updated_at": "2026-01-01T12:00:00Z", "tags": ["factory", "edge-node"] } }""" tokenizer_path = "meta-llama/Meta-Llama-3-8B-Instruct" tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) grammar_compiler = xgr.GrammarCompiler(tokenizer_info, cache_enabled=True) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) compiled_grammar_a = grammar_compiler.compile_json_schema(schema_a) compiled_grammar_b = grammar_compiler.compile_json_schema(schema_b) input_bytes_a = string_a.encode("utf-8") matcher_a = xgr.GrammarMatcher(compiled_grammar_a) input_bytes_b = string_b.encode("utf-8") matcher_b = xgr.GrammarMatcher(compiled_grammar_b) rejected_sizes = [] for i, c in enumerate(input_bytes_a): matcher_a.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask( token_bitmask, tokenizer_info.vocab_size ) rejected_sizes.append(len(rejected_token_ids)) assert rejected_sizes[-1] == rejected_a[i], (rejected_sizes[-1], rejected_a[i]) assert matcher_a.accept_string(bytes([c])) matcher_a.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size) rejected_sizes.append(len(rejected_token_ids)) assert rejected_sizes[-1] == rejected_a[-1] rejected_sizes = [] for i, c in enumerate(input_bytes_b): matcher_b.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask( token_bitmask, tokenizer_info.vocab_size ) rejected_sizes.append(len(rejected_token_ids)) assert rejected_sizes[-1] == rejected_b[i], (rejected_sizes[-1], rejected_b[i]) assert matcher_b.accept_string(bytes([c])) matcher_b.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size) rejected_sizes.append(len(rejected_token_ids)) assert rejected_sizes[-1] == rejected_b[-1] if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.2.3/tests/python/test_grammar_matcher_macro.py000066400000000000000000000176001521764210300237450ustar00rootroot00000000000000import sys import pytest import xgrammar as xgr from xgrammar.testing import _get_masked_tokens_from_bitmask, _is_grammar_accept_string def test_simple(): grammar_str = """root ::= TagDispatch(("tag1", rule1), ("tag2", rule2)) rule1 ::= "abcd" rule2 ::= "efg" """ grammar = xgr.Grammar.from_ebnf(grammar_str) assert _is_grammar_accept_string(grammar, "tag1abcd") assert _is_grammar_accept_string(grammar, "tag1abcdtag2efg") assert _is_grammar_accept_string(grammar, "tag1abcdqqqqtag2efg") assert not _is_grammar_accept_string(grammar, "tag1abc") assert not _is_grammar_accept_string(grammar, "tag1abce") assert not _is_grammar_accept_string(grammar, "ttag1abd") def test_complex_rule(): grammar_str = """root ::= TagDispatch(("tag1", rule1), ("tag2", rule2)) rule1 ::= "abcd" [p]* rule2 ::= "efg" [t]* """ grammar = xgr.Grammar.from_ebnf(grammar_str) assert _is_grammar_accept_string(grammar, "tag1abcd") assert _is_grammar_accept_string(grammar, "tag1abcdppppptag2efg") assert _is_grammar_accept_string(grammar, "tag2efgtttttag1abc") assert not _is_grammar_accept_string(grammar, "tag1efg") def test_no_loop_after_dispatch(): grammar_str = """root ::= TagDispatch(("tag1", rule1), ("tag2", rule2), loop_after_dispatch=false) rule1 ::= "abcd" [p]* rule2 ::= "efg" [t]* """ grammar = xgr.Grammar.from_ebnf(grammar_str) assert _is_grammar_accept_string(grammar, "tag1abcd") assert _is_grammar_accept_string(grammar, "tag2efgttt") assert not _is_grammar_accept_string(grammar, "tag1abcdppppptag2") assert not _is_grammar_accept_string(grammar, "tag2efgtag1") def test_stop_str(): grammar_str = """root ::= root1 stop "w" root1 ::= TagDispatch( ("tag1", rule1), ("tag2", rule2), excludes=("tag3", "ll") ) stop ::= "tag3" | "ll" rule1 ::= "abcd" [p]* rule2 ::= "efg" [t]* """ grammar = xgr.Grammar.from_ebnf(grammar_str) assert _is_grammar_accept_string(grammar, "tag1abcdllw", debug_print=True) assert _is_grammar_accept_string(grammar, "tag1abcdtag3w") assert _is_grammar_accept_string(grammar, "tag1abcdqqqtag2efgtag3w") assert _is_grammar_accept_string(grammar, "tag1abcd", require_termination=False) assert _is_grammar_accept_string(grammar, "tag2efgttt", require_termination=False) assert not _is_grammar_accept_string(grammar, "tag1abcd") assert not _is_grammar_accept_string(grammar, "tag2efgttt") assert not _is_grammar_accept_string(grammar, "tag1abce") assert not _is_grammar_accept_string(grammar, "tag1abcdlltag3w", require_termination=False) def test_stop_str_no_loop(): grammar_str = """root ::= root1 stop "w" root1 ::= TagDispatch( ("tag1", rule1), ("tag2", rule2), excludes=("tag3", "ll"), loop_after_dispatch=false ) stop ::= "tag3" | "ll" rule1 ::= "abcd" [p]* rule2 ::= "efg" [t]* """ grammar = xgr.Grammar.from_ebnf(grammar_str) assert _is_grammar_accept_string(grammar, "tag1abcdllw") assert _is_grammar_accept_string(grammar, "tag1abcdtag3w") assert _is_grammar_accept_string(grammar, "tag1abcd", require_termination=False) assert _is_grammar_accept_string(grammar, "tag2efgttt", require_termination=False) assert not _is_grammar_accept_string(grammar, "tag1abcdqqqtag2efgtag3w") assert not _is_grammar_accept_string(grammar, "tag1abcd") assert not _is_grammar_accept_string(grammar, "tag2efgttt") assert not _is_grammar_accept_string(grammar, "tag1abce") assert not _is_grammar_accept_string(grammar, "tag1abcdlltag3w", require_termination=False) def test_tag_dispatch_mask_generation_correctness(): grammar_str = """root ::= TagDispatch(("tag1", rule1), ("tag2", rule2)) rule1 ::= "abc" rule2 ::= "dg" """ tokens = [ # fmt: off "a", "b", "c", "d", "g", "t", "1", "2", "1a", "2d", "2a", "2dgt", "2dgtag1a", "2dgtag1b", "tag1a", "tag1b", "c哈哈t", "q", "abcdef" # fmt: on ] input_str = "tag1abcqqtag2dgq" expected_accepted_tokens = [ # fmt: off ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2a', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'], ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2a', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'], ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2a', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'], ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'], ['a', 'abcdef'], ['b'], ['c哈哈t', 'c'], ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2a', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'], ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2a', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'], ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2a', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'], ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2a', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'], ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2a', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'], ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'], ['d'], ['g'], ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2a', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'], ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2a', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'] # fmt: on ] grammar = xgr.Grammar.from_ebnf(grammar_str) tokenizer_info = xgr.TokenizerInfo(tokens) compiler = xgr.GrammarCompiler(tokenizer_info, max_threads=1) compiled_grammar = compiler.compile_grammar(grammar) matcher = xgr.GrammarMatcher(compiled_grammar, terminate_without_stop_token=True) mask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) # pad a dummy char to check the final bitmask after accepting the input string for i, c in enumerate(input_str + "0"): matcher.fill_next_token_bitmask(mask) rejected_indices = _get_masked_tokens_from_bitmask(mask, tokenizer_info.vocab_size) accepted_indices = list(set(range(tokenizer_info.vocab_size)) - set(rejected_indices)) accepted_tokens = [tokens[id] for id in accepted_indices] if i < len(input_str): assert matcher.accept_string(c) assert accepted_tokens == expected_accepted_tokens[i] def test_regression_multiple_tag_dispatch(): grammar_str = """root ::= root1 "w" root1 ::= TagDispatch( ("tag1", rule1), ("tag2", rule2), loop_after_dispatch=false ) rule1 ::= rule1_dispatch rule1_stop rule1_dispatch ::= TagDispatch( ("tag1", rule2), ("tag2", rule3), excludes=("tag3", "ll"), loop_after_dispatch=true ) rule1_stop ::= "tag3" | "ll" rule2 ::= "efg" [t]* rule3 ::= "abcd" [p]* """ assert _is_grammar_accept_string(grammar_str, "tag1tag1efgllw") assert _is_grammar_accept_string(grammar_str, "tag1tag2abcdtag3w") assert not _is_grammar_accept_string(grammar_str, "tag1Ktag2abcdtag3tag1") assert _is_grammar_accept_string(grammar_str, "tag1tag3w") assert not _is_grammar_accept_string(grammar_str, "tag1tag3tag2abcdll") def test_excluded_str(): grammar_str = """root ::= root_dispatch end_tag root_dispatch ::= TagDispatch( ("start", rule1), excludes=("", ""), loop_after_dispatch=true ) rule1 ::= "12345" end_tag ::= "" """ grammar = xgr.Grammar.from_ebnf(grammar_str) printed = str(grammar) assert 'excludes=("", "")' in printed assert _is_grammar_accept_string(grammar, "start12345") assert not _is_grammar_accept_string(grammar, "start12345") assert _is_grammar_accept_string(grammar, "start12345abc") assert not _is_grammar_accept_string(grammar, "start12345abc") if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.2.3/tests/python/test_grammar_matcher_regex.py000066400000000000000000000162301521764210300237540ustar00rootroot00000000000000import sys import time import pytest import torch from transformers import AutoTokenizer import xgrammar as xgr from xgrammar.testing import _get_masked_tokens_from_bitmask, _is_grammar_accept_string def test_simple(): regex_str = "abc" grammar = xgr.Grammar.from_regex(regex_str) assert _is_grammar_accept_string(grammar, "abc") assert not _is_grammar_accept_string(grammar, "ab") assert not _is_grammar_accept_string(grammar, "abcd") test_repetition_input_accepted_test_repetition = ( ("aaa", True), ("abcbc", True), ("bcbcbcbcbc", True), ("bcbcbcbcbcbcbcb", True), ("d", False), ("aaaa", False), ) @pytest.mark.parametrize("input, accepted", test_repetition_input_accepted_test_repetition) def test_repetition(input: str, accepted: bool): regex_str = "(a|[bc]{4,}){2,3}" grammar = xgr.Grammar.from_regex(regex_str) assert _is_grammar_accept_string(grammar, input) == accepted test_regex_accept_regex_input_accepted = [ r"abc", r"[abc]+", r"[a-z0-9]+", r"[^abc]+", r"a*b+c?", r"(abc|def)+", r"a{2,4}", r"\d+", r"\w+", r"[A-Z][a-z]*", r"[0-9]{3}-[0-9]{3}-[0-9]{4}", r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", ] @pytest.mark.parametrize("regex_input_accepted", test_regex_accept_regex_input_accepted) def test_regex_accept(regex_input_accepted: str): grammar = xgr.Grammar.from_regex(regex_input_accepted) assert grammar is not None test_regex_refuse_regex_input_refused = ( r"a{,3}", # Invalid range r"a{3,2}", # Invalid range (max < min) r"[z-a]", # Invalid range (max < min) r"a++", # Invalid repetition r"(?=a)", # Lookahead not supported r"(?!a)", # Negative lookahead not supported ) @pytest.mark.parametrize("regex_input_refused", test_regex_refuse_regex_input_refused) def test_regex_refuse(regex_input_refused: str): with pytest.raises(RuntimeError): xgr.Grammar.from_regex(regex_input_refused) test_advanced_regex_string_instance_is_accepted = [ # Basic patterns (r"abc", "abc", True), (r"abc", "def", False), # Character classes (r"[abc]+", "aabbcc", True), (r"[abc]+", "abcd", False), (r"[a-z0-9]+", "abc123", True), (r"[a-z0-9]+", "ABC", False), (r"[^abc]+", "def", True), (r"[^abc]+", "aaa", False), # Lazy character class (r"[abc]+?abc", "aabc", True), # Quantifiers (r"a*b+c?", "b", True), (r"a*b+c?", "aaabbc", True), (r"a*b+c?", "c", False), # Alternation (r"(abc|def)+", "abcdef", True), (r"(abc|def)+", "abcabc", True), (r"(abc|def)+", "ab", False), # Repetition ranges (r"a{2,4}", "aa", True), (r"a{2,4}", "aaaa", True), (r"a{2,4}", "a", False), (r"a{2,4}", "aaaaa", False), # Common patterns (r"\d+", "123", True), (r"\d+", "abc", False), (r"\w+", "abc123", True), (r"\w+", "!@#", False), (r"[A-Z][a-z]*", "Hello", True), (r"[A-Z][a-z]*", "hello", False), # Complex patterns (r"[0-9]{3}-[0-9]{3}-[0-9]{4}", "123-456-7890", True), (r"[0-9]{3}-[0-9]{3}-[0-9]{4}", "12-34-567", False), (r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}", "test@email.com", True), (r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}", "invalid.email", False), ] @pytest.mark.parametrize( "regex_string, instance, is_accepted", test_advanced_regex_string_instance_is_accepted ) def test_advanced(regex_string: str, instance: str, is_accepted: bool): grammar = xgr.Grammar.from_regex(regex_string) assert _is_grammar_accept_string(grammar, instance) == is_accepted regex_input_str_test_fill_next_token_bitmask = [ (r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}", "test@email.com"), (r"[0-9]{3}-[0-9]{3}-[0-9]{4}", "123-456-7890"), ] @pytest.mark.hf_token_required @pytest.mark.parametrize("regex, input_str", regex_input_str_test_fill_next_token_bitmask) def test_fill_next_token_bitmask(regex: str, input_str: str): tokenizer_path = "meta-llama/Meta-Llama-3-8B-Instruct" tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) time_start = time.monotonic_ns() compiled_grammar = compiler.compile_regex(regex) matcher = xgr.GrammarMatcher(compiled_grammar) time_end = time.monotonic_ns() print(f"Time to init GrammarMatcher: {(time_end - time_start) / 1e3} us") input_bytes = input_str.encode("utf-8") token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) for c in input_bytes: time_start = time.monotonic_ns() assert matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time to fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") time_start = time.monotonic_ns() assert matcher.accept_string(bytes([c])) time_end = time.monotonic_ns() print(f"Time to accept char {chr(c)}: {(time_end - time_start) / 1e3} us") matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size) assert tokenizer.eos_token_id not in rejected_token_ids @pytest.mark.hf_token_required def test_regex_with_large_range_compilation(): regex_with_large_range = r"[a-z]{100,20000}" tokenizer_path = "meta-llama/Meta-Llama-3-8B-Instruct" tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) time_start = time.monotonic_ns() _ = compiler.compile_regex(regex_with_large_range) time_end = time.monotonic_ns() print(f"Time to compile regex with large range: {(time_end - time_start) / 1e3} us") @pytest.mark.hf_token_required def test_regression_lookahead_already_completed(): tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-0.5B") tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) xgr_compiler = xgr.GrammarCompiler(tokenizer_info, max_threads=1) compiled_grammar = xgr_compiler.compile_regex(r"\/\*(\*+[^*\/]|[^*])*\*+\/") matcher = xgr.GrammarMatcher(compiled_grammar) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) def process_logit(input_ids: list, logit: torch.Tensor) -> torch.Tensor: if input_ids: last_token = input_ids[-1] assert matcher.accept_token(last_token) matcher.fill_next_token_bitmask(token_bitmask) xgr.apply_token_bitmask_inplace(logit, token_bitmask) return logit def process_tokens(tokens: list): for i in range(len(tokens)): logit = torch.zeros((tokenizer_info.vocab_size,), dtype=torch.float) visible_tokens = tokens[:i] masked_logit = process_logit(visible_tokens, logit) assert masked_logit[tokens[i]] != float( "-inf" ), f"token {i} ({tokens[i]}, {tokenizer.decode(tokens[i])!r}) is masked" text = "/* */" tokens = tokenizer.encode(text) process_tokens(tokens) if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.2.3/tests/python/test_grammar_matcher_structural_tag.py000066400000000000000000000574141521764210300257160ustar00rootroot00000000000000import json import sys import threading import time from typing import List import pytest from pydantic import BaseModel from transformers import AutoTokenizer import xgrammar as xgr from xgrammar.testing import _get_masked_tokens_from_bitmask, _is_grammar_accept_string def test_utf8(): # Test utf8-encoded string with structural tags class Schema(BaseModel): arg1: str arg2: int tags = [ xgr.StructuralTagItem(begin="，，", schema=Schema, end="。"), xgr.StructuralTagItem(begin="，！", schema=Schema, end="。。"), xgr.StructuralTagItem(begin="，，？", schema=Schema, end="。。。"), xgr.StructuralTagItem(begin="｜｜？", schema=Schema, end="｜？｜"), ] triggers = ["，", "｜｜"] grammar = xgr.Grammar.from_structural_tag(tags, triggers) accepted_inputs = [ '这是无用的内容，，{"arg1": "你好，世界！", "arg2": 0}。这是无用的内容', '这是无用的内容，！{"arg1": "こんにちは！", "arg2": 1}。。这是无用的内容', '这是无用的内容，，？{"arg1": "안녕하세요！", "arg2": 2}。。。这是无用的内容，！{"arg1": "안녕하세요！", "arg2": 3}。。', '这是无用的内容｜｜？{"arg1": "။စ်န, ်ပြ！", "arg2": 0}｜？｜｜｜？{"arg1": "။စ်န, ်ပြ", "arg2": 0}｜？｜', ] for input_str in accepted_inputs: assert _is_grammar_accept_string(grammar, input_str, print_time=True) expected_grammar_test_structural_tag_after_optimization = r"""basic_escape ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) (=(basic_string_sub)) basic_string_sub ::= (("\"") | ([^\0-\x1f\"\\\r\n] basic_string_sub) | ("\\" basic_escape basic_string_sub)) (=([ \n\t]* [,}\]:])) basic_integer ::= (("0") | (basic_integer_1 [1-9] [0-9]*)) basic_string ::= (("\"" basic_string_sub)) (=(root_part_0 [ \n\t]* "}")) root_part_0 ::= (([ \n\t]* "," [ \n\t]* "\"arg2\"" [ \n\t]* ":" [ \n\t]* basic_integer)) (=([ \n\t]* "}")) root_0 ::= (("{" [ \n\t]* "\"arg1\"" [ \n\t]* ":" [ \n\t]* basic_string root_part_0 [ \n\t]* "}")) basic_integer_1 ::= ("" | ("-")) (=([1-9] [0-9]*)) basic_escape_1 ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) (=(basic_string_sub_1)) basic_string_sub_1 ::= (("\"") | ([^\0-\x1f\"\\\r\n] basic_string_sub_1) | ("\\" basic_escape_1 basic_string_sub_1)) (=([ \n\t]* [,}\]:])) basic_number_8 ::= ((basic_number_1_1 basic_number_7_1 basic_number_3_1 basic_number_6_1)) (=(root_part_0_1 [ \n\t]* "}")) basic_string_1 ::= (("\"" basic_string_sub_1)) root_prop_1 ::= (("[" [ \n\t]* basic_string_1 root_prop_1_1 [ \n\t]* "]") | ("[" [ \n\t]* "]")) root_part_0_1 ::= (([ \n\t]* "," [ \n\t]* "\"arg4\"" [ \n\t]* ":" [ \n\t]* root_prop_1)) (=([ \n\t]* "}")) root_1 ::= (("{" [ \n\t]* "\"arg3\"" [ \n\t]* ":" [ \n\t]* basic_number_8 root_part_0_1 [ \n\t]* "}")) (=("")) basic_number_1_1 ::= ("" | ("-")) (=(basic_number_7_1 basic_number_3_1 basic_number_6_1)) basic_number_2_1 ::= (([0-9] basic_number_2_1) | ([0-9])) basic_number_3_1 ::= ("" | ("." basic_number_2_1)) (=(basic_number_6_1)) basic_number_4_1 ::= ("" | ([+\-])) (=(basic_number_5_1)) basic_number_5_1 ::= (([0-9] basic_number_5_1) | ([0-9])) basic_number_6_1 ::= ("" | ([eE] basic_number_4_1 basic_number_5_1)) root_prop_1_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_string_1 root_prop_1_1)) (=([ \n\t]* "]")) basic_number_7_1 ::= (("0") | ([1-9] [0-9]*)) (=(basic_number_3_1 basic_number_6_1)) triggered_tags_group ::= (("1>" root_0 "") | ("2>" root_0 "")) triggered_tags_group_1 ::= ((">" root_1 "")) triggered_tags ::= TagDispatch( ("" root_0 "") | ("2>" root_0 "")) triggered_tags_group_1 ::= ((">" root_1 "")) triggered_tags ::= TagDispatch( ("{"arg1": "abc", "arg2": 1}', '{"arg3": 1.23, "arg4": ["a", "b", "c"]}', '{"arg1": "abc", "arg2": 1}{"arg3": 1.23, "arg4": ["a", "b", "c"]}', 'hhhh{"arg3": 1.23, "arg4": ["a", "b", "c"]}haha{"arg1": "abc", "arg2": 1}123', ] for input in accepted_inputs: assert _is_grammar_accept_string(grammar, input, print_time=True) def test_structural_tag_compiler(): class Schema1(BaseModel): arg1: str arg2: int class Schema2(BaseModel): arg3: float arg4: List[str] tags = [ xgr.StructuralTagItem(begin="", schema=Schema1, end=""), xgr.StructuralTagItem(begin="", schema=Schema1, end=""), xgr.StructuralTagItem(begin="", schema=Schema2, end=""), ] # in real cases, we should use one trigger: "{"arg3": 1.23, "arg4": ["a", "b", "c"]}' 'haha{"arg1": "abc", "arg2": 1}123' ) dont_apply_mask_indices = [ # fmt: off 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 119, 120, 121, 122 # fmt: on ] input_bytes = accepted_input.encode("utf-8") # Set up token bitmask for validation token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) # Process input character by character for i, c in enumerate(input_bytes): # 1. Test token bitmask generation time_start = time.monotonic_ns() need_apply = matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time to fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") assert need_apply == (i not in dont_apply_mask_indices) # 2. Verify token bitmask correctness rejected_token_ids = _get_masked_tokens_from_bitmask( token_bitmask, tokenizer_info.vocab_size ) # This checking does not support non-ascii characters for now token_id_for_next_char = tokenizer.convert_tokens_to_ids(chr(c)) assert token_id_for_next_char not in rejected_token_ids # 3. Test character acceptance print("Accepting char:", bytes([c])) time_start = time.monotonic_ns() assert matcher.accept_string(bytes([c])) time_end = time.monotonic_ns() print(f"Time to accept_token: {(time_end - time_start) / 1e3} us") # Final verification - check that EOS token is allowed time_start = time.monotonic_ns() need_apply = matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() assert need_apply == (len(input_bytes) not in dont_apply_mask_indices) print(f"Time to fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") rejected_token_ids = _get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size) assert tokenizer.eos_token_id not in rejected_token_ids def test_empty_tag_dispatch(): grammar_str = """root ::= TagDispatch( loop_after_dispatch=true ) """ grammar = xgr.Grammar.from_ebnf(grammar_str) assert _is_grammar_accept_string(grammar, "any string") assert _is_grammar_accept_string(grammar, "") assert _is_grammar_accept_string(grammar, "好") grammar_with_excludes_str = """root ::= TagDispatch( excludes=("end"), loop_after_dispatch=true ) """ grammar_with_excludes = xgr.Grammar.from_ebnf(grammar_with_excludes_str) assert _is_grammar_accept_string(grammar_with_excludes, "any string") assert _is_grammar_accept_string(grammar_with_excludes, "好") assert not _is_grammar_accept_string(grammar_with_excludes, "any stringend") assert not _is_grammar_accept_string(grammar_with_excludes, "endaaa") @pytest.mark.hf_token_required def test_utf8_structural_tag_begin_end(): model = "deepseek-ai/DeepSeek-V3-0324" tokenizer = AutoTokenizer.from_pretrained(model) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) structures = [ xgr.StructuralTagItem(begin="<｜tool▁calls▁begin｜>", schema={}, end="<｜tool▁calls▁end｜>") ] triggers = ["<｜tool▁calls▁begin｜>"] _ = compiler.compile_structural_tag(structures, triggers) @pytest.mark.hf_token_required def test_pressure_structural_tag(): model = "meta-llama/Llama-3.1-8B-Instruct" tokenizer = AutoTokenizer.from_pretrained(model, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info, max_threads=1) threads = [] start = "start" schema = {"type": "object", "properties": {"arg": {"type": "string"}}} end = "end" def worker(idx: int): tag = xgr.StructuralTagItem(begin=start, schema=schema, end=end) triggers = [start] stag_grammar = xgr.Grammar.from_structural_tag([tag], triggers) start_grammar = xgr.Grammar.from_ebnf("root ::= [a-z] root | [a-z]") grammar = start_grammar for _ in range(idx): grammar = grammar.concat(grammar, start_grammar) final_grammar = xgr.Grammar.concat(grammar, stag_grammar) _ = compiler.compile_grammar(final_grammar) for i in range(128): t = threading.Thread(target=worker, args=(i,)) threads.append(t) t.start() for t in threads: t.join() _JSON_BODY_RULES = """\ json_body ::= "{" ws kvs ws "}" kvs ::= kv ("," ws kv)* kv ::= ["] key_chars ["] ws ":" ws val key_chars ::= [a-zA-Z_] [a-zA-Z0-9_]* val ::= ["] val_chars ["] | [0-9]+ | "true" | "false" val_chars ::= [a-zA-Z0-9 _.+/=]* ws ::= [ ]* """ _tag_dispatch_perf_scenarios = [ # S1: Long exclude, normal text (baseline - most tokens hit fast path) ( f"""root ::= TagDispatch(("", json_body), loop_after_dispatch=true, excludes=("")) {_JSON_BODY_RULES}""", ( "The quick brown fox jumps over the lazy dog. " "Machine learning models have revolutionized natural language processing. " "Transformers use self-attention mechanisms to capture long-range dependencies. " '{"action": "search", "query": "test"}' "After retrieving results we can summarize the findings effectively. " "The experiment showed significant improvements across all benchmarks measured." ), ), # S2: Short exclude "\n\n" (many tokens contain \n -> second_slicing_bitset fails often) ( f"""root ::= TagDispatch(("", json_body), loop_after_dispatch=true, excludes=("\\n\\n")) {_JSON_BODY_RULES}""", ( "def hello():\n print('hello')\n" "def world():\n return 42\n" "class Foo:\n def bar(self):\n pass\n" "x = [i for i in range(10)]\n" "result = sum(x)\n" '{"action": "run"}' "for item in collection:\n process(item)\n" "logger.info('done')\n" ), ), # S3: Single char exclude "|" (extreme short exclude) ( f"""root ::= TagDispatch(("", json_body), loop_after_dispatch=true, excludes=("|")) {_JSON_BODY_RULES}""", ( "This is a simple text without any pipe characters in the content. " "We keep writing more content to have enough tokens for measurement. " "The test validates single character exclude performance overhead. " '{"tag": "content", "value": "here"}' "More text after the tag to continue the sequence to the end." ), ), # S4: Dense partial match (FSM repeatedly enters exclude prefix -> slow path) ( f"""root ::= TagDispatch(("", json_body), loop_after_dispatch=true, excludes=("")) {_JSON_BODY_RULES}""", ( "Check ", tc), ("", ts), ("", tcode), ("", tf), ("", tw), ("", tdb), ("", tapi), ("", tsh), loop_after_dispatch=true, excludes=("") ) tc ::= json_body ts ::= json_body tcode ::= json_body tf ::= json_body tw ::= json_body tdb ::= json_body tapi ::= json_body tsh ::= json_body {_JSON_BODY_RULES}""", ( "Here is some text before any tags appear in the output. " '{"expr": "2+3"}' "The answer is 5. Now let me search for more information. " '{"query": "xgrammar benchmarks"}' "Found some results. Let me write code to process them. " '{"code": "print hello"}' "Code executed successfully. Now checking files on disk. " '{"path": "data.csv"}' "Data loaded. Processing complete with all results verified." ), ), # S6: 8 exclude strings (more excludes -> more tokens hit substring check) ( f"""root ::= TagDispatch( ("", json_body), loop_after_dispatch=true, excludes=("", "STOP", "HALT", "QUIT", "EXIT", "ABORT", "CANCEL", "TERMINATE") ) {_JSON_BODY_RULES}""", ( "Starting the process now. The system is running fine and stable. " "Several steps are being executed in proper sequence order. " "Have you seen the strategy that was described in the handbook? " '{"action": "compute", "value": 42}' "The tool returned a value. Continuing execution of the pipeline. " "After checking everything looks correct and verified properly." ), ), # S7: 10 tag dispatch loop cycles (tests loop overhead) ( f"""root ::= TagDispatch(("", json_body), loop_after_dispatch=true, excludes=("")) {_JSON_BODY_RULES}""", ( 'A{"k": "v1"}B{"k": "v2"}C{"k": "v3"}D{"k": "v4"}E{"k": "v5"}' 'F{"k": "v6"}G{"k": "v7"}H{"k": "v8"}I{"k": "v9"}J{"k": "v10"}K' ), ), # S8: No tags, pure AnyText (lightest TagDispatch, reference baseline) ( """root ::= TagDispatch(loop_after_dispatch=false, excludes=("")) """, ( "This is purely text without any tags at all in the output. " "We are testing the lightest possible TagDispatch configuration. " "No tags are defined, only a single exclude string is present. " "This serves as the baseline reference measurement for comparison." ), ), # S9: Sustained exclude boundary (FSM stays in exclude prefix -> all tokens slow path) ( f"""root ::= TagDispatch(("", json_body), loop_after_dispatch=true, excludes=("")) {_JSON_BODY_RULES}""", ( "Normal text. = warmup compiler = xgr.GrammarCompiler(tokenizer_info, cache_enabled=False) t0 = time.monotonic_ns() compiled = compiler.compile_grammar(grammar) t1 = time.monotonic_ns() if is_measure: compile_times.append((t1 - t0) / 1e6) matcher = xgr.GrammarMatcher(compiled, terminate_without_stop_token=True) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) round_mask = [] round_accept = [] for tok in input_tokens: t_m0 = time.monotonic_ns() matcher.fill_next_token_bitmask(token_bitmask) t_m1 = time.monotonic_ns() t_a0 = time.monotonic_ns() ok = matcher.accept_token(tok) t_a1 = time.monotonic_ns() if is_measure: round_mask.append((t_m1 - t_m0) / 1e3) round_accept.append((t_a1 - t_a0) / 1e3) assert ok, f"token {tok} ({repr(tokenizer.decode([tok]))}) rejected" if is_measure: mask_times_all.append(round_mask) accept_times_all.append(round_accept) flat_mask = [t for rnd in mask_times_all for t in rnd] flat_accept = [t for rnd in accept_times_all for t in rnd] print(f"Tokens: {len(input_tokens)}") print( f"Compile: {statistics.mean(compile_times):.2f} +/- " f"{statistics.stdev(compile_times) if len(compile_times) > 1 else 0:.2f} ms" ) print(f"Mask gen: avg={statistics.mean(flat_mask):.2f} us, max={max(flat_mask):.2f} us") print(f"Accept token: avg={statistics.mean(flat_accept):.2f} us, max={max(flat_accept):.2f} us") if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.2.3/tests/python/test_grammar_parser.py000066400000000000000000000751231521764210300224410ustar00rootroot00000000000000import sys from typing import Optional import pytest import xgrammar as xgr from xgrammar.testing import GrammarFunctor, _ebnf_to_grammar_no_normalization def test_basic_string_literal(): """Test basic string literals in grammar rules.""" before = """root ::= "hello" """ expected = """root ::= (("hello")) """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_empty_string(): """Test empty string literals.""" before = """root ::= "" """ expected = """root ::= (("")) """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_character_class(): """Test character class expressions.""" before = """root ::= [a-z] """ expected = """root ::= (([a-z])) """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_negated_character_class(): """Test negated character class expressions.""" before = """root ::= [^a-z] """ expected = """root ::= (([^a-z])) """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_complex_character_class(): """Test complex character class with multiple ranges and individual characters.""" before = r"""root ::= [a-zA-Z0-9_-] [\r\n$\x10-o\]\--] """ expected = r"""root ::= (([a-zA-Z0-9_\-] [\r\n$\x10-o\]\-\-])) """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_sequence(): """Test sequence of expressions.""" before = """root ::= "a" "b" "c" """ expected = """root ::= (("a" "b" "c")) """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_choice(): """Test choice between expressions.""" before = """root ::= "a" | "b" | "c" """ expected = """root ::= (("a") | ("b") | ("c")) """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_grouping(): """Test grouping with parentheses.""" before = """root ::= ("a" "b") | ("c" "d") """ expected = """root ::= (((("a" "b"))) | ((("c" "d")))) """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_star_quantifier_simple(): """Test star (*) quantifier.""" before = """root ::= "a"* """ expected = """root ::= ((root_1)) root_1 ::= ("" | ("a" root_1)) """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_plus_quantifier(): """Test plus (+) quantifier.""" before = """root ::= "a"+ """ expected = """root ::= ((root_1)) root_1 ::= (("a" root_1) | "a") """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_question_quantifier(): """Test question (?) quantifier.""" before = """root ::= "a"? """ expected = """root ::= ((root_1)) root_1 ::= ("" | "a") """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_character_class_star(): """Test star (*) quantifier with character class.""" before = """root ::= [a-z]* """ expected = """root ::= (([a-z]*)) """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_repetition_range_exact(): """Test repetition range with exact count {n}.""" before = """root ::= "a"{3} """ expected = """root ::= ((root_1{3, 3})) root_1 ::= "a" """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_repetition_range_min_max(): """Test repetition range with min and max {n,m}.""" before = """root ::= "a"{2,4} """ expected = """root ::= ((root_1{2, 4})) root_1 ::= "a" """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_repetition_range_min_only(): """Test repetition range with only min {n,}.""" before = """root ::= "a"{2,} """ expected = """root ::= ((root_1{2, -1})) root_1 ::= "a" """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_repetition_range_unbounded_roundtrip(): """Printed {n, -1} can be re-parsed (str -> compile_grammar round-trip).""" before = """root ::= "a"{2,} """ grammar_1 = xgr.Grammar.from_ebnf(before) output_1 = str(grammar_1) assert "{2, -1}" in output_1 output_2 = str(xgr.Grammar.from_ebnf(output_1)) assert output_1 == output_2 def test_repetition_range_unbounded_json_schema(): """JSON schema minLength produces {n, -1} which round-trips through the parser.""" import json schema = json.dumps({"type": "string", "minLength": 2}) grammar_1 = xgr.Grammar.from_json_schema(schema) output_1 = str(grammar_1) assert "{2, -1}" in output_1 output_2 = str(xgr.Grammar.from_ebnf(output_1)) assert output_1 == output_2 def test_lookahead_assertion_simple(): """Test lookahead assertion.""" before = """root ::= "a" (="b") """ expected = """root ::= (("a")) (=(("b"))) """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_complex_lookahead(): """Test complex lookahead assertion.""" before = """root ::= "a" (="b" "c" [0-9]) """ expected = """root ::= (("a")) (=(("b" "c" [0-9]))) """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_escape_sequences(): """Test escape sequences in string literals.""" before = r"""root ::= "\n\t\r\"\\" """ expected = r"""root ::= (("\n\t\r\"\\")) """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_unicode_escape(): """Test Unicode escape sequences.""" before = r"""root ::= "\u0041\u0042\u0043\u00A9\u2603" """ expected = r"""root ::= (("ABC\xa9\u2603")) """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_forward_slash_escape_in_string_literal(): # Regression: the EBNF lexer used to reject "\/" in string literals # with "Invalid escape sequence", because the C-style escape table did # not include "/". JSON allows "\/" as an alias for "/", so xgrammar # should accept it consistently. before = r"""root ::= "a\/b" """ expected = r"""root ::= (("a/b")) """ grammar = _ebnf_to_grammar_no_normalization(before) assert str(grammar) == expected def test_complex_grammar(): """Test a more complex grammar with multiple features.""" before = """root ::= expr expr ::= term ("+" term | "-" term)* term ::= factor ("*" factor | "/" factor)* factor ::= number | "(" expr ")" number ::= [0-9]+ ("." [0-9]+)? """ expected = """root ::= ((expr)) expr ::= ((term expr_1)) term ::= ((factor term_1)) factor ::= ((number) | ("(" expr ")")) number ::= ((number_1 number_3)) expr_1 ::= ("" | ((("+" term) | ("-" term)) expr_1)) term_1 ::= ("" | ((("*" factor) | ("/" factor)) term_1)) number_1 ::= (([0-9] number_1) | [0-9]) number_2 ::= (([0-9] number_2) | [0-9]) number_3 ::= ("" | (("." number_2))) """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_nested_quantifiers(): """Test nested quantifiers in expressions.""" before = """root ::= ("a"*)+ """ expected = """root ::= ((root_2)) root_1 ::= ("" | ("a" root_1)) root_2 ::= ((((root_1)) root_2) | ((root_1))) """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_combined_features(): """Test combination of various grammar features.""" before = """root ::= "start" (rule1 | rule2)+ "end" rule1 ::= [a-z]{1,3} (=":") rule2 ::= [0-9]+ "." [0-9]* """ expected = """root ::= (("start" root_1 "end")) rule1 ::= ((rule1_1{1, 3})) (=((":"))) rule2 ::= ((rule2_1 "." [0-9]*)) root_1 ::= ((((rule1) | (rule2)) root_1) | ((rule1) | (rule2))) rule1_1 ::= [a-z] rule2_1 ::= (([0-9] rule2_1) | [0-9]) """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_bnf_comment(): before = """# top comment root ::= a b # inline comment a ::= "a" b ::= "b" # bottom comment """ expected = """root ::= ((a b)) a ::= (("a")) b ::= (("b")) """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_star_quantifier(): before = """root ::= b c d b ::= [b]* c ::= "b"* d ::= ([b] [c] [d] | ([p] [q]))* e ::= [e]* [f]* | [g]* """ expected = """root ::= ((b c d)) b ::= (([b]*)) c ::= ((c_1)) d ::= ((d_1)) e ::= (([e]* [f]*) | ([g]*)) c_1 ::= ("" | ("b" c_1)) d_1 ::= ("" | (d_1_1 d_1)) d_1_1 ::= (("b" "c" "d") | ("p" "q")) """ grammar = _ebnf_to_grammar_no_normalization(before) grammar = GrammarFunctor.structure_normalizer(grammar) after = str(grammar) assert after == expected # Here rule1 can be empty before = """root ::= [a]* [b]* rule1 rule1 ::= [abc]* [def]* """ expected = """root ::= (([a]* [b]* rule1)) rule1 ::= (([abc]* [def]*)) """ grammar = _ebnf_to_grammar_no_normalization(before) grammar = GrammarFunctor.structure_normalizer(grammar) after = str(grammar) assert after == expected def test_repetition_range(): before = """root ::= a b c d e f g a ::= [a]{1,2} b ::= (a | "b"){1, 5} c ::= "c" {0 , 2} d ::= "d" {0,} e ::= "e" {2, } f ::= "f" {3} g ::= "g" {0} """ expected = """root ::= ((a b c d e f g)) a ::= ((a_1{1, 2})) b ::= ((b_1{1, 5})) c ::= ((c_1{0, 2})) d ::= ((d_1{0, -1})) e ::= ((e_1{2, -1})) f ::= ((f_1{3, 3})) g ::= ((g_1{0, 0})) a_1 ::= (("a")) b_1 ::= ((a) | ("b")) c_1 ::= (("c")) d_1 ::= (("d")) e_1 ::= (("e")) f_1 ::= (("f")) g_1 ::= (("g")) """ grammar = _ebnf_to_grammar_no_normalization(before) grammar = GrammarFunctor.structure_normalizer(grammar) after = str(grammar) assert after == expected def test_lookahead_assertion_with_normalizer(): before = """root ::= ((b c d)) b ::= (("abc" [a-z])) (=("abc")) c ::= (("a") | ("b")) (=[a-z] "b") d ::= (("ac") | ("b" d_choice)) (="abc") d_choice ::= (("e") | ("d")) """ expected = """root ::= ((b c d)) b ::= (("abc" [a-z])) (=("abc")) c ::= (("a") | ("b")) (=([a-z] "b")) d ::= (("ac") | ("b" d_choice)) (=("abc")) d_choice ::= (("e") | ("d")) """ grammar = _ebnf_to_grammar_no_normalization(before) grammar = GrammarFunctor.structure_normalizer(grammar) after = str(grammar) assert after == expected def test_char(): before = r"""root ::= [a-z] [A-z] "\u0234" "\U00000345\xff" [-A-Z] [--] [^a] rest rest ::= [a-zA-Z0-9-] [\u0234-\U00000345] [测-试] [\--\]] rest1 rest1 ::= "\?\"\'测试あc" "👀" "" [a-a] [b-b] """ expected = r"""root ::= (([a-z] [A-z] "\u0234" "\u0345\xff" [\-A-Z] [\-\-] [^a] rest)) rest ::= (([a-zA-Z0-9\-] [\u0234-\u0345] [\u6d4b-\u8bd5] [\--\]] rest1)) rest1 ::= (("\?\"\'\u6d4b\u8bd5\u3042c" "\U0001f440" "a" "b")) """ # Disable unwrap_nesting_rules to expose the result before unwrapping. grammar = _ebnf_to_grammar_no_normalization(before) grammar = GrammarFunctor.structure_normalizer(grammar) after = str(grammar) assert after == expected def test_space(): before = """ root::="a" "b" ("c""d" "e") | "f" | "g" """ expected = """root ::= (("a" "b" "c" "d" "e") | ("f") | ("g")) """ grammar = xgr.Grammar.from_ebnf(before) after = str(grammar) assert after == expected def test_nest(): before = """root::= "a" ("b" | "c" "d") | (("e" "f")) """ expected = """root ::= (("a" root_1) | ("e" "f")) root_1 ::= (("b") | ("c" "d")) """ grammar = xgr.Grammar.from_ebnf(before) after = str(grammar) assert after == expected def test_empty_parentheses(): before = """root ::= "a" ( ) "b" """ expected = """root ::= (("a" "b")) """ grammar = xgr.Grammar.from_ebnf(before) after = str(grammar) assert after == expected before = """root ::= "a" rule1 rule1 ::= ( ) """ expected = """root ::= (("a" rule1)) rule1 ::= ("") """ grammar = xgr.Grammar.from_ebnf(before) after = str(grammar) assert after == expected def test_lookahead_assertion_analyzer(): before = r"""root ::= "a" rule1 "b" rule3 rule5 rule2 rule1 ::= "b" rule2 ::= "c" rule3 ::= "" | "d" rule3 rule4 ::= "" | "e" rule4 "f" rule5 ::= "" | "g" rule5 "h" """ expected = r"""root ::= (("a" rule1 "b" rule3 rule5 rule2)) rule1 ::= (("b")) (=("b" rule3 rule5 rule2)) rule2 ::= (("c")) rule3 ::= (("") | ("d" rule3)) (=(rule5 rule2)) rule4 ::= (("") | ("e" rule4 "f")) (=("f")) rule5 ::= (("") | ("g" rule5 "h")) """ grammar = _ebnf_to_grammar_no_normalization(before) grammar = GrammarFunctor.lookahead_assertion_analyzer(grammar) after = str(grammar) assert after == expected def test_flatten(): before = """root ::= or_test sequence_test nested_test empty_test or_test ::= ([a] | "b") | "de" | "" | or_test | [^a-z] sequence_test ::= [a] "a" ("b" ("c" | "d")) ("d" "e") sequence_test "" nested_test ::= ("a" ("b" ("c" "d"))) | ("a" | ("b" | "c")) | nested_rest nested_rest ::= ("a" | ("b" "c" | ("d" | "e" "f"))) | ((("g"))) empty_test ::= "d" | (("" | "" "") "" | "a" "") | ("" ("" | "")) "" "" """ expected = """root ::= ((or_test sequence_test nested_test empty_test)) or_test ::= ("" | ("a") | ("b") | ("de") | (or_test) | ([^a-z])) sequence_test ::= (("a" "a" "b" sequence_test_1 "d" "e" sequence_test)) nested_test ::= (("a" "b" "c" "d") | ("a") | ("b") | ("c") | (nested_rest)) nested_rest ::= (("a") | ("b" "c") | ("d") | ("e" "f") | ("g")) empty_test ::= ("" | ("d") | ("a")) sequence_test_1 ::= (("c") | ("d")) """ grammar = _ebnf_to_grammar_no_normalization(before) grammar = GrammarFunctor.structure_normalizer(grammar) after = str(grammar) assert after == expected before__expected__test_rule_inliner = [ ( r"""root ::= rule1 | rule2 rule1 ::= "a" | "b" rule2 ::= "b" | "c" """, r"""root ::= (("a") | ("b") | ("b") | ("c")) rule1 ::= (("a") | ("b")) rule2 ::= (("b") | ("c")) """, ), ( r"""root ::= rule1 "a" [a-z]* | rule2 "b" "c" rule1 ::= "a" [a-z]* | "b" rule2 ::= "b" | "c" [b-c] """, r"""root ::= (("a" [a-z]* "a" [a-z]*) | ("b" "a" [a-z]*) | ("b" "b" "c") | ("c" [b-c] "b" "c")) rule1 ::= (("a" [a-z]*) | ("b")) rule2 ::= (("b") | ("c" [b-c])) """, ), ] @pytest.mark.parametrize("before, expected", before__expected__test_rule_inliner) def test_rule_inliner(before: str, expected: str): grammar = _ebnf_to_grammar_no_normalization(before) grammar = GrammarFunctor.rule_inliner(grammar) after = str(grammar) assert after == expected before__expected__test_dead_code_eliminator = [ # Test basic dead code elimination ( r"""root ::= rule1 | rule2 rule1 ::= "a" | "b" rule2 ::= "b" | "c" unused ::= "x" | "y" """, r"""root ::= ((rule1) | (rule2)) rule1 ::= (("a") | ("b")) rule2 ::= (("b") | ("c")) """, ), # Test recursive rule references ( r"""root ::= rule1 | rule2 unused1 ::= unused2 | "x" unused2 ::= unused1 | "y" rule1 ::= "a" rule2 | "b" rule2 ::= "c" rule1 | "d" """, r"""root ::= ((rule1) | (rule2)) rule1 ::= (("a" rule2) | ("b")) rule2 ::= (("c" rule1) | ("d")) """, ), # Test complex nested rules with unused branches ( r"""root ::= rule1 "x" | rule2 rule1 ::= "a" rule3 | "b" rule2 ::= "c" | "d" rule4 rule3 ::= "e" | "f" rule4 ::= "g" | "h" unused1 ::= "i" unused2 unused2 ::= "j" unused3 unused3 ::= "k" | "l" """, r"""root ::= ((rule1 "x") | (rule2)) rule1 ::= (("a" rule3) | ("b")) rule2 ::= (("c") | ("d" rule4)) rule3 ::= (("e") | ("f")) rule4 ::= (("g") | ("h")) """, ), ] @pytest.mark.parametrize("before, expected", before__expected__test_dead_code_eliminator) def test_dead_code_eliminator(before: str, expected: str): grammar = _ebnf_to_grammar_no_normalization(before) after = xgr.testing.GrammarFunctor.dead_code_eliminator(grammar) assert str(after) == expected def test_e2e_json_grammar(): before = r"""root ::= ( "{" [ \n\t]* members_and_embrace | "[" [ \n\t]* elements_or_embrace ) value_non_str ::= ( "{" [ \n\t]* members_and_embrace | "[" [ \n\t]* elements_or_embrace | "0" fraction exponent | [1-9] [0-9]* fraction exponent | "-" [0-9] fraction exponent | "-" [1-9] [0-9]* fraction exponent | "true" | "false" | "null" ) (= [ \n\t,}\]]) members_and_embrace ::= ("\"" characters_and_colon [ \n\t]* members_suffix | "}") (= [ \n\t,}\]]) members_suffix ::= ( value_non_str [ \n\t]* member_suffix_suffix | "\"" characters_and_embrace | "\"" characters_and_comma [ \n\t]* "\"" characters_and_colon [ \n\t]* members_suffix ) (= [ \n\t,}\]]) member_suffix_suffix ::= ( "}" | "," [ \n\t]* "\"" characters_and_colon [ \n\t]* members_suffix ) (= [ \n\t,}\]]) elements_or_embrace ::= ( "{" [ \n\t]* members_and_embrace elements_rest [ \n\t]* "]" | "[" [ \n\t]* elements_or_embrace elements_rest [ \n\t]* "]" | "\"" characters_item elements_rest [ \n\t]* "]" | "0" fraction exponent elements_rest [ \n\t]* "]" | [1-9] [0-9]* fraction exponent elements_rest [ \n\t]* "]" | "-" "0" fraction exponent elements_rest [ \n\t]* "]" | "-" [1-9] [0-9]* fraction exponent elements_rest [ \n\t]* "]" | "true" elements_rest [ \n\t]* "]" | "false" elements_rest [ \n\t]* "]" | "null" elements_rest [ \n\t]* "]" | "]" ) elements ::= ( "{" [ \n\t]* members_and_embrace elements_rest | "[" [ \n\t]* elements_or_embrace elements_rest | "\"" characters_item elements_rest | "0" fraction exponent elements_rest | [1-9] [0-9]* fraction exponent elements_rest | "-" [0-9] fraction exponent elements_rest | "-" [1-9] [0-9]* fraction exponent elements_rest | "true" elements_rest | "false" elements_rest | "null" elements_rest ) elements_rest ::= ( "" | [ \n\t]* "," [ \n\t]* elements ) characters_and_colon ::= ( "\"" [ \n\t]* ":" | [^"\\\x00-\x1F] characters_and_colon | "\\" escape characters_and_colon ) (=[ \n\t]* [\"{[0-9tfn-]) characters_and_comma ::= ( "\"" [ \n\t]* "," | [^"\\\x00-\x1F] characters_and_comma | "\\" escape characters_and_comma ) (=[ \n\t]* "\"") characters_and_embrace ::= ( "\"" [ \n\t]* "}" | [^"\\\x00-\x1F] characters_and_embrace | "\\" escape characters_and_embrace ) (=[ \n\t]* [},]) characters_item ::= ( "\"" | [^"\\\x00-\x1F] characters_item | "\\" escape characters_item ) (= [ \n\t]* [,\]]) escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] fraction ::= "" | "." [0-9] [0-9]* exponent ::= "" | "e" sign [0-9] [0-9]* | "E" sign [0-9] [0-9]* sign ::= "" | "+" | "-" """ expected = r"""root ::= (("{" [ \n\t]* members_and_embrace) | ("[" [ \n\t]* elements_or_embrace)) value_non_str ::= (("{" [ \n\t]* members_and_embrace) | ("[" [ \n\t]* elements_or_embrace) | ("0" fraction exponent) | ([1-9] [0-9]* fraction exponent) | ("-" [0-9] fraction exponent) | ("-" [1-9] [0-9]* fraction exponent) | ("true") | ("false") | ("null")) (=([ \n\t,}\]])) members_and_embrace ::= (("\"" characters_and_colon [ \n\t]* members_suffix) | ("}")) (=([ \n\t,}\]])) members_suffix ::= ((value_non_str [ \n\t]* member_suffix_suffix) | ("\"" characters_and_embrace) | ("\"" characters_and_comma [ \n\t]* "\"" characters_and_colon [ \n\t]* members_suffix)) (=([ \n\t,}\]])) member_suffix_suffix ::= (("}") | ("," [ \n\t]* "\"" characters_and_colon [ \n\t]* members_suffix)) (=([ \n\t,}\]])) elements_or_embrace ::= (("{" [ \n\t]* members_and_embrace elements_rest [ \n\t]* "]") | ("[" [ \n\t]* elements_or_embrace elements_rest [ \n\t]* "]") | ("\"" characters_item elements_rest [ \n\t]* "]") | ("0" fraction exponent elements_rest [ \n\t]* "]") | ([1-9] [0-9]* fraction exponent elements_rest [ \n\t]* "]") | ("-0" fraction exponent elements_rest [ \n\t]* "]") | ("-" [1-9] [0-9]* fraction exponent elements_rest [ \n\t]* "]") | ("true" elements_rest [ \n\t]* "]") | ("false" elements_rest [ \n\t]* "]") | ("null" elements_rest [ \n\t]* "]") | ("]")) elements ::= (("{" [ \n\t]* members_and_embrace elements_rest) | ("[" [ \n\t]* elements_or_embrace elements_rest) | ("\"" characters_item elements_rest) | ("0" fraction exponent elements_rest) | ([1-9] [0-9]* fraction exponent elements_rest) | ("-" [0-9] fraction exponent elements_rest) | ("-" [1-9] [0-9]* fraction exponent elements_rest) | ("true" elements_rest) | ("false" elements_rest) | ("null" elements_rest)) elements_rest ::= ("" | ([ \n\t]* "," [ \n\t]* elements)) characters_and_colon ::= (("\"" [ \n\t]* ":") | ([^\"\\\0-\x1f] characters_and_colon) | ("\\" escape characters_and_colon)) (=([ \n\t]* [\"{[0-9tfn\-])) characters_and_comma ::= (("\"" [ \n\t]* ",") | ([^\"\\\0-\x1f] characters_and_comma) | ("\\" escape characters_and_comma)) (=([ \n\t]* "\"")) characters_and_embrace ::= (("\"" [ \n\t]* "}") | ([^\"\\\0-\x1f] characters_and_embrace) | ("\\" escape characters_and_embrace)) (=([ \n\t]* [},])) characters_item ::= (("\"") | ([^\"\\\0-\x1f] characters_item) | ("\\" escape characters_item)) (=([ \n\t]* [,\]])) escape ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) fraction ::= ("" | ("." [0-9] [0-9]*)) exponent ::= ("" | ("e" sign [0-9] [0-9]*) | ("E" sign [0-9] [0-9]*)) sign ::= ("" | ("+") | ("-")) """ grammar = xgr.Grammar.from_ebnf(before) grammar = GrammarFunctor.grammar_optimizer(grammar) after = str(grammar) assert after == expected def test_e2e_to_string_roundtrip(): """Checks the printed result can be parsed, and the parsing-printing process is idempotent.""" before = r"""root ::= ((b c) | (b root)) b ::= ((b_1 d)) c ::= ((c_1)) d ::= ((d_1)) b_1 ::= ("" | ("b" b_1)) (=(d)) c_1 ::= (([acep-z] c_1) | ([acep-z])) (=("d")) d_1 ::= ("" | ("d")) """ grammar_1 = xgr.Grammar.from_ebnf(before) output_string_1 = str(grammar_1) grammar_2 = xgr.Grammar.from_ebnf(output_string_1) output_string_2 = str(grammar_2) assert before == output_string_1 assert output_string_1 == output_string_2 ebnf_str__expected_error_regex__test_lexer_parser_errors = [ (r'root ::= "a" "', 'EBNF lexer error at line 1, column 15: Expect " in string literal'), ( "root ::= [a\n]", "EBNF lexer error at line 1, column 12: Character class should not contain newline", ), (r'root ::= "\@"', "EBNF lexer error at line 1, column 11: Invalid escape sequence"), (r'root ::= "\uFF"', "EBNF lexer error at line 1, column 11: Invalid escape sequence"), (r'::= "a"', "EBNF lexer error at line 1, column 1: Assign should not be the first token"), (r"root ::= a b", 'EBNF parser error at line 1, column 10: Rule "a" is not defined'), (r'root ::= "a" |', "EBNF parser error at line 1, column 15: Expect element"), ( r"root ::= [Z-A]", "EBNF parser error at line 1, column 11: Invalid character class: lower bound is larger " "than upper bound", ), ( 'root ::= "a"\nroot ::= "b"', 'EBNF parser error at line 2, column 1: Rule "root" is defined multiple times', ), ( r'a ::= "a"', 'EBNF parser error at line 1, column 1: The root rule with name "root" is not found', ), (r'root ::= "a" (="a") (="b")', "EBNF parser error at line 1, column 21: Expect rule name"), ] @pytest.mark.parametrize( "ebnf_str, expected_error_regex", ebnf_str__expected_error_regex__test_lexer_parser_errors ) def test_lexer_parser_errors(ebnf_str: str, expected_error_regex: Optional[str]): with pytest.raises(RuntimeError, match=expected_error_regex): _ebnf_to_grammar_no_normalization(ebnf_str) ebnf_str__expected_error_regex__test_end_to_end_errors = [ (r'root ::= "a" (=("a" | "b"))', "Choices in lookahead assertion are not supported yet") ] @pytest.mark.parametrize( "ebnf_str, expected_error_regex", ebnf_str__expected_error_regex__test_end_to_end_errors ) def test_end_to_end_errors(ebnf_str: str, expected_error_regex: Optional[str]): with pytest.raises(RuntimeError, match=expected_error_regex): xgr.Grammar.from_ebnf(ebnf_str) def test_error_consecutive_quantifiers(): grammar_str = """root ::= "a"{1,3}{1,3} """ with pytest.raises( RuntimeError, match="EBNF parser error at line 1, column 18: Expect element, but got {" ): xgr.Grammar.from_ebnf(grammar_str) grammar_str = """root ::= "a"++ """ with pytest.raises( RuntimeError, match="EBNF parser error at line 1, column 14: Expect element, but got +" ): xgr.Grammar.from_ebnf(grammar_str) grammar_str = """root ::= "a"?? """ with pytest.raises( RuntimeError, match="EBNF parser error at line 1, column 14: Expect element, but got ?" ): xgr.Grammar.from_ebnf(grammar_str) def test_repetition_normalizer(): """Test the repetition normalizer. If the context is nullable, then the min repetition time will be reduced to 0.""" before = "root ::= ([0-9]*){200, 1000}" expected_grammar = r"""root ::= ((root_2)) root_repeat_1 ::= (([0-9]*)) (=([0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]*)) root_repeat_1_inner ::= ((root_repeat_1{0, 872})) (=([0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]*)) root_2 ::= ((root_repeat_1_inner [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]* [0-9]*)) """ grammar = xgr.Grammar.from_ebnf(before) print(grammar) grammar = GrammarFunctor.grammar_optimizer(grammar) assert expected_grammar == str(grammar) before = "root ::= ([0-9]){200, 1000}" expected_grammar = r"""root ::= ((root_2)) root_repeat_1 ::= (([0-9])) (=([0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9])) root_repeat_1_inner ::= ((root_repeat_1{72, 872})) (=([0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9])) root_2 ::= ((root_repeat_1_inner [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9] [0-9])) """ grammar = xgr.Grammar.from_ebnf(before) grammar = GrammarFunctor.grammar_optimizer(grammar) assert expected_grammar == str(grammar) if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.2.3/tests/python/test_grammar_parser_macro.py000066400000000000000000000134061521764210300236160ustar00rootroot00000000000000"""Tests the macro features of the grammar parser.""" import sys from typing import Optional import pytest import xgrammar as xgr from xgrammar.testing import GrammarFunctor, _ebnf_to_grammar_no_normalization def test_tag_dispatch(): """Test TagDispatch functionality.""" before = """root ::= TagDispatch( ("tag1", rule1), ("tag2", rule2), excludes = ("abc", "def"), loop_after_dispatch = false ) rule1 ::= "a" rule2 ::= "b" """ expected = """root ::= ((TagDispatch( ("tag1", rule1), ("tag2", rule2), loop_after_dispatch=false, excludes=("abc", "def") ))) rule1 ::= (("a")) rule2 ::= (("b")) """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_tag_dispatch_default_parameters(): """Test TagDispatch functionality.""" before = """root ::= TagDispatch(("tag1", rule1), ("tag2", rule2)) rule1 ::= "a" rule2 ::= "b" """ expected = """root ::= ((TagDispatch( ("tag1", rule1), ("tag2", rule2), loop_after_dispatch=true, excludes=() ))) rule1 ::= (("a")) rule2 ::= (("b")) """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_lookahead_assertion_analyzer_tag_dispatch(): # tag dispatch disables lookahead assertion detection before = r"""root ::= TagDispatch(("tag1", rule1), ("tag2", rule2), ("tag3", rule3), ("tag4", rule4), ("tag5", rule5)) rule1 ::= "b" rule2 ::= "c" rule3 ::= "" | "d" rule3 rule4 ::= "" | "e" rule4 "f" rule5 ::= "" | "g" rule5 "h" """ expected = r"""root ::= TagDispatch( ("tag1", rule1), ("tag2", rule2), ("tag3", rule3), ("tag4", rule4), ("tag5", rule5), loop_after_dispatch=true, excludes=() ) rule1 ::= (("b")) rule2 ::= (("c")) rule3 ::= ("" | ("d" rule3)) rule4 ::= ("" | ("e" rule4 "f")) rule5 ::= ("" | ("g" rule5 "h")) """ grammar = _ebnf_to_grammar_no_normalization(before) grammar = GrammarFunctor.structure_normalizer(grammar) grammar = GrammarFunctor.byte_string_fuser(grammar) grammar = GrammarFunctor.lookahead_assertion_analyzer(grammar) after = str(grammar) assert after == expected def test_tag_dispatch_end_to_end(): before = """root ::= TagDispatch(("tag1", rule1), ("tag2", rule2), ("tag3", rule3)) rule1 ::= "a" rule2 ::= "b" rule3 ::= "c" """ expected = """root ::= TagDispatch( ("tag1", rule1), ("tag2", rule2), ("tag3", rule3), loop_after_dispatch=true, excludes=() ) rule1 ::= (("a")) rule2 ::= (("b")) rule3 ::= (("c")) """ grammar = xgr.Grammar.from_ebnf(before) after = str(grammar) assert after == expected def test_tag_dispatch_end_to_end_complex(): before = """root ::= TagDispatch(("tag1", rule1), ("tag2", rule2), ("tag3", rule3)) rule1 ::= ("a" TagDispatch(("tag1", rule2), ("tag2", rule3)) | "zzz") rule2 ::= TagDispatch(("tag1", rule2), ("tag2", rule3)) | TagDispatch(("tag3", rule2), ("tag4", rule3)) rule3 ::= "c" """ expected = """root ::= TagDispatch( ("tag1", rule1), ("tag2", rule2), ("tag3", rule3), loop_after_dispatch=true, excludes=() ) rule1 ::= (("a" rule1_1) | ("zzz")) rule2 ::= ((rule2_1) | (rule2_2)) rule3 ::= (("c")) rule1_1 ::= TagDispatch( ("tag1", rule2), ("tag2", rule3), loop_after_dispatch=true, excludes=() ) rule2_1 ::= TagDispatch( ("tag1", rule2), ("tag2", rule3), loop_after_dispatch=true, excludes=() ) rule2_2 ::= TagDispatch( ("tag3", rule2), ("tag4", rule3), loop_after_dispatch=true, excludes=() ) """ grammar = xgr.Grammar.from_ebnf(before) after = str(grammar) assert after == expected def test_e2e_tag_dispatch_roundtrip(): """Checks the printed result can be parsed, and the parsing-printing process is idempotent.""" before = r"""root ::= TagDispatch( ("tag1", rule1), ("tag2", rule2), ("tag3", rule3), loop_after_dispatch=false, excludes=() ) rule1 ::= (("a")) rule2 ::= (("b")) rule3 ::= (("c")) """ grammar_1 = xgr.Grammar.from_ebnf(before) output_string_1 = str(grammar_1) grammar_2 = xgr.Grammar.from_ebnf(output_string_1) output_string_2 = str(grammar_2) assert before == output_string_1 assert output_string_1 == output_string_2 ebnf_str__expected_error_regex__test_tag_dispatch_parser_errors = [ ( 'root ::= TagDispatch(("", rule1))\nrule1 ::= "a"', "EBNF parser error at line 1, column 21: Tag must be a non-empty string literal", ), ( 'root ::= TagDispatch(("tag1", undefined_rule))', 'EBNF parser error at line 1, column 21: Rule "undefined_rule" is not defined', ), ( 'root ::= TagDispatch("tag1", rule1)', "EBNF parser error at line 1, column 21: Each tag dispatch element must be a tuple", ), ( 'root ::= TagDispatch(("tag1" rule1))', "EBNF parser error at line 1, column 30: Expect , or \\) in tuple", ), ( 'root ::= TagDispatch(("tag1", rule1), stop_str=true)\nrule1 ::= "a"', "EBNF parser error at line 1, column 21: Unknown named argument for TagDispatch: stop_str", ), ( 'root ::= TagDispatch(("tag1", rule1), stop_eos=false)\nrule1 ::= "a"', "EBNF parser error at line 1, column 21: Unknown named argument for TagDispatch: stop_eos", ), ( 'root ::= TagDispatch(("tag1", rule1), excludes=("tag1"))\nrule1 ::= "a"', "EBNF parser error at line 1, column 21: Exclude string must not be a prefix of trigger string: tag1", ), ] @pytest.mark.parametrize( "ebnf_str, expected_error_regex", ebnf_str__expected_error_regex__test_tag_dispatch_parser_errors, ) def test_tag_dispatch_parser_errors(ebnf_str: str, expected_error_regex: Optional[str]): with pytest.raises(RuntimeError, match=expected_error_regex): _ebnf_to_grammar_no_normalization(ebnf_str) if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.2.3/tests/python/test_grammar_union_concat.py000066400000000000000000000134251521764210300236210ustar00rootroot00000000000000"""This test uses the optimized JSON grammar provided by the grammar library.""" import sys import pytest import xgrammar as xgr def test_grammar_union(): grammar1 = xgr.Grammar.from_ebnf( """root ::= r1 | r2 r1 ::= "true" | "" r2 ::= "false" | "" """ ) grammar2 = xgr.Grammar.from_ebnf( """root ::= "abc" | r1 r1 ::= "true" | r1 """ ) grammar3 = xgr.Grammar.from_ebnf( """root ::= r1 | r2 | r3 r1 ::= "true" | r3 r2 ::= "false" | r3 r3 ::= "abc" | "" """ ) expected = """root ::= ((root_1) | (root_2) | (root_3)) root_1 ::= ((r1) | (r2)) r1 ::= ("" | ("true")) r2 ::= ("" | ("false")) root_2 ::= (("abc") | (r1_1)) r1_1 ::= (("true") | (r1_1)) root_3 ::= ((r1_2) | (r2_1) | (r3)) r1_2 ::= (("true") | (r3)) r2_1 ::= (("false") | (r3)) r3 ::= ("" | ("abc")) """ union_grammar = xgr.Grammar.union(grammar1, grammar2, grammar3) assert str(union_grammar) == expected def test_grammar_concat(): grammar1 = xgr.Grammar.from_ebnf( """root ::= r1 | r2 r1 ::= "true" | "" r2 ::= "false" | "" """ ) grammar2 = xgr.Grammar.from_ebnf( """root ::= "abc" | r1 r1 ::= "true" | r1 """ ) grammar3 = xgr.Grammar.from_ebnf( """root ::= r1 | r2 | r3 r1 ::= "true" | r3 r2 ::= "false" | r3 r3 ::= "abc" | "" """ ) expected = """root ::= ((root_1 root_2 root_3)) root_1 ::= ((r1) | (r2)) r1 ::= ("" | ("true")) r2 ::= ("" | ("false")) root_2 ::= (("abc") | (r1_1)) r1_1 ::= (("true") | (r1_1)) root_3 ::= ((r1_2) | (r2_1) | (r3)) r1_2 ::= (("true") | (r3)) r2_1 ::= (("false") | (r3)) r3 ::= ("" | ("abc")) """ concat_grammar = xgr.Grammar.concat(grammar1, grammar2, grammar3) assert str(concat_grammar) == expected def test_grammar_union_with_stag(): expected_grammar_union = r"""root ::= ((root_1) | (root_2)) basic_escape ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) basic_string_sub ::= (("\"") | ([^\0-\x1f\"\\\r\n] basic_string_sub) | ("\\" basic_escape basic_string_sub)) (=([ \n\t]* [,}\]:])) basic_any ::= ((basic_number) | (basic_string) | (basic_boolean) | (basic_null) | (basic_array) | (basic_object)) basic_integer ::= (("0") | (basic_integer_1 [1-9] [0-9]*)) basic_number ::= ((basic_number_1 basic_number_7 basic_number_3 basic_number_6)) basic_string ::= (("\"" basic_string_sub)) basic_boolean ::= (("true") | ("false")) basic_null ::= (("null")) basic_array ::= (("[" [ \n\t]* basic_any basic_array_1 [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= (("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1 [ \n\t]* "}") | ("{" [ \n\t]* "}")) root_0 ::= (("{" [ \n\t]* "\"arg\"" [ \n\t]* ":" [ \n\t]* basic_string [ \n\t]* "}") | ("{" [ \n\t]* "}")) basic_integer_1 ::= ("" | ("-")) basic_number_1 ::= ("" | ("-")) basic_number_2 ::= (([0-9] basic_number_2) | ([0-9])) basic_number_3 ::= ("" | ("." basic_number_2)) basic_number_4 ::= ("" | ([+\-])) basic_number_5 ::= (([0-9] basic_number_5) | ([0-9])) basic_number_6 ::= ("" | ([eE] basic_number_4 basic_number_5)) basic_array_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_any basic_array_1)) basic_object_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1)) basic_number_7 ::= (("0") | ([1-9] [0-9]*)) triggered_tags_group ::= (("" root_0 "end")) triggered_tags ::= TagDispatch( ("start", triggered_tags_group), loop_after_dispatch=true, excludes=() ) root_1 ::= ((triggered_tags)) root_2 ::= (([a-z] root_2) | ([a-z])) """ expected_grammar_concat = r"""root ::= ((root_1 root_2)) basic_escape ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) basic_string_sub ::= (("\"") | ([^\0-\x1f\"\\\r\n] basic_string_sub) | ("\\" basic_escape basic_string_sub)) (=([ \n\t]* [,}\]:])) basic_any ::= ((basic_number) | (basic_string) | (basic_boolean) | (basic_null) | (basic_array) | (basic_object)) basic_integer ::= (("0") | (basic_integer_1 [1-9] [0-9]*)) basic_number ::= ((basic_number_1 basic_number_7 basic_number_3 basic_number_6)) basic_string ::= (("\"" basic_string_sub)) basic_boolean ::= (("true") | ("false")) basic_null ::= (("null")) basic_array ::= (("[" [ \n\t]* basic_any basic_array_1 [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= (("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1 [ \n\t]* "}") | ("{" [ \n\t]* "}")) root_0 ::= (("{" [ \n\t]* "\"arg\"" [ \n\t]* ":" [ \n\t]* basic_string [ \n\t]* "}") | ("{" [ \n\t]* "}")) basic_integer_1 ::= ("" | ("-")) basic_number_1 ::= ("" | ("-")) basic_number_2 ::= (([0-9] basic_number_2) | ([0-9])) basic_number_3 ::= ("" | ("." basic_number_2)) basic_number_4 ::= ("" | ([+\-])) basic_number_5 ::= (([0-9] basic_number_5) | ([0-9])) basic_number_6 ::= ("" | ([eE] basic_number_4 basic_number_5)) basic_array_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_any basic_array_1)) basic_object_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1)) basic_number_7 ::= (("0") | ([1-9] [0-9]*)) triggered_tags_group ::= (("" root_0 "end")) triggered_tags ::= TagDispatch( ("start", triggered_tags_group), loop_after_dispatch=true, excludes=() ) root_1 ::= ((triggered_tags)) root_2 ::= (([a-z] root_2) | ([a-z])) """ start = "start" schema = {"type": "object", "properties": {"arg": {"type": "string"}}} end = "end" tag = xgr.StructuralTagItem(begin=start, schema=schema, end=end) triggers = [start] stag_grammar = xgr.Grammar.from_structural_tag([tag], triggers) start_grammar = xgr.Grammar.from_ebnf("root ::= [a-z] root | [a-z]") grammar_union = xgr.Grammar.union(stag_grammar, start_grammar) assert str(grammar_union) == expected_grammar_union grammar_concat = xgr.Grammar.concat(stag_grammar, start_grammar) assert str(grammar_concat) == expected_grammar_concat if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.2.3/tests/python/test_json_schema_converter.py000066400000000000000000004040001521764210300240050ustar00rootroot00000000000000import json import re import sys from enum import Enum from typing import Any, Dict, List, Literal, Optional, Tuple, Type, Union import pytest from pydantic import BaseModel, Field, TypeAdapter, create_model import xgrammar as xgr from xgrammar.testing import ( GrammarFunctor, _generate_float_regex, _generate_range_regex, _is_grammar_accept_string, _json_schema_to_ebnf, ) basic_json_rules_ebnf = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" """ basic_json_rules_ebnf_no_space = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" "" basic_any (", " basic_any)* "" "]") | ("[" "" "]")) basic_object ::= ("{" "" basic_string ": " basic_any (", " basic_string ": " basic_any)* "" "}") | "{" "}" """ def check_schema_with_grammar( schema: Dict[str, Any], expected_grammar_ebnf: str, any_whitespace: bool = True, indent: Optional[int] = None, separators: Optional[Tuple[str, str]] = None, strict_mode: bool = True, ): json_schema_ebnf = _json_schema_to_ebnf( schema, any_whitespace=any_whitespace, indent=indent, separators=separators, strict_mode=strict_mode, ) assert json_schema_ebnf == expected_grammar_ebnf def check_schema_with_instance( schema: Dict[str, Any], instance: Union[str, BaseModel, Any], is_accepted: bool = True, any_whitespace: bool = True, indent: Optional[int] = None, separators: Optional[Tuple[str, str]] = None, strict_mode: bool = True, debug_print: bool = False, ): json_schema_grammar = xgr.Grammar.from_json_schema( json.dumps(schema), any_whitespace=any_whitespace, indent=indent, separators=separators, strict_mode=strict_mode, ) # instance: pydantic model, json string, or any other object (dumped to json string) if isinstance(instance, BaseModel): instance = json.dumps( instance.model_dump(mode="json", round_trip=True), indent=indent, separators=separators ) elif not isinstance(instance, str): instance = json.dumps(instance, indent=indent, separators=separators) accepted = _is_grammar_accept_string(json_schema_grammar, instance, debug_print=debug_print) assert accepted == is_accepted def test_basic(): class MainModel(BaseModel): integer_field: int number_field: float boolean_field: bool any_array_field: List array_field: List[str] tuple_field: Tuple[str, int, List[str]] object_field: Dict[str, int] nested_object_field: Dict[str, Dict[str, int]] ebnf_grammar = basic_json_rules_ebnf_no_space + ( r"""root_prop_3 ::= (("[" "" basic_any (", " basic_any)* "" "]") | ("[" "" "]")) root_prop_4 ::= (("[" "" basic_string (", " basic_string)* "" "]") | ("[" "" "]")) root_prop_5_item_2 ::= (("[" "" basic_string (", " basic_string)* "" "]") | ("[" "" "]")) root_prop_5 ::= ("[" "" (basic_string ", " basic_integer ", " root_prop_5_item_2) "" "]") root_prop_6 ::= ("{" "" basic_string ": " basic_integer (", " basic_string ": " basic_integer)* "" "}") | "{" "}" root_prop_7_addl ::= ("{" "" basic_string ": " basic_integer (", " basic_string ": " basic_integer)* "" "}") | "{" "}" root_prop_7 ::= ("{" "" basic_string ": " root_prop_7_addl (", " basic_string ": " root_prop_7_addl)* "" "}") | "{" "}" root_part_6 ::= ", " "\"nested_object_field\"" ": " root_prop_7 "" root_part_5 ::= ", " "\"object_field\"" ": " root_prop_6 root_part_6 root_part_4 ::= ", " "\"tuple_field\"" ": " root_prop_5 root_part_5 root_part_3 ::= ", " "\"array_field\"" ": " root_prop_4 root_part_4 root_part_2 ::= ", " "\"any_array_field\"" ": " root_prop_3 root_part_3 root_part_1 ::= ", " "\"boolean_field\"" ": " basic_boolean root_part_2 root_part_0 ::= ", " "\"number_field\"" ": " basic_number root_part_1 root ::= "{" "" (("\"integer_field\"" ": " basic_integer root_part_0)) "" "}" """ ) schema = MainModel.model_json_schema() check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=False) instance = MainModel( integer_field=42, number_field=3.14e5, boolean_field=True, any_array_field=[3.14, "foo", None, True], array_field=["foo", "bar"], tuple_field=("foo", 42, ["bar", "baz"]), object_field={"foo": 42, "bar": 43}, nested_object_field={"foo": {"bar": 42}}, ) check_schema_with_instance(schema, instance, any_whitespace=False) def test_indent(): class MainModel(BaseModel): array_field: List[str] tuple_field: Tuple[str, int, List[str]] object_field: Dict[str, int] ebnf_grammar = basic_json_rules_ebnf_no_space + ( r"""root_prop_0 ::= (("[" "\n " basic_string (",\n " basic_string)* "\n " "]") | ("[" "" "]")) root_prop_1_item_2 ::= (("[" "\n " basic_string (",\n " basic_string)* "\n " "]") | ("[" "" "]")) root_prop_1 ::= ("[" "\n " (basic_string ",\n " basic_integer ",\n " root_prop_1_item_2) "\n " "]") root_prop_2 ::= ("{" "\n " basic_string ": " basic_integer (",\n " basic_string ": " basic_integer)* "\n " "}") | "{" "}" root_part_1 ::= ",\n " "\"object_field\"" ": " root_prop_2 "" root_part_0 ::= ",\n " "\"tuple_field\"" ": " root_prop_1 root_part_1 root ::= "{" "\n " (("\"array_field\"" ": " root_prop_0 root_part_0)) "\n" "}" """ ) instance = MainModel( array_field=["foo", "bar"], tuple_field=("foo", 42, ["bar", "baz"]), object_field={"foo": 42, "bar": 43}, ) schema = MainModel.model_json_schema() check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=False, indent=2) check_schema_with_instance(schema, instance, any_whitespace=False, indent=2) check_schema_with_instance( schema, instance, any_whitespace=False, indent=None, separators=(",", ":") ) schema__grammar__accepted_instances__rejected_instances__test_non_strict = [ ( {"type": "array", "prefixItems": [{"type": "integer"}, {"type": "integer"}]}, basic_json_rules_ebnf + r"""root_additional ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object root ::= ("[" [ \n\t]* (basic_integer [ \n\t]* "," [ \n\t]* basic_integer) ([ \n\t]* "," [ \n\t]* root_additional)* [ \n\t]* "]") """, [[1, 2], [1, 2, 3], [1, 2, 3, "123"]], [[1]], ), ( { "type": "object", "properties": {"foo": {"type": "integer"}, "bar": {"type": "integer"}}, "required": ["foo", "bar"], }, basic_json_rules_ebnf + r"""root_addl ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object root_addl_key ::= ["] (("\"" | [^bf\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "b" ("\"" | [^a\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "a" ("\"" | [^r\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "r" ([^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub))) | "f" ("\"" | [^o\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "o" ("\"" | [^o\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "o" ([^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub))))) (= [ \n\t]* [,}\]:]) root_part_1 ::= ([ \n\t]* "," [ \n\t]* root_addl_key [ \n\t]* ":" [ \n\t]* root_addl)* root_part_0 ::= [ \n\t]* "," [ \n\t]* "\"bar\"" [ \n\t]* ":" [ \n\t]* basic_integer root_part_1 root ::= "{" [ \n\t]* (("\"foo\"" [ \n\t]* ":" [ \n\t]* basic_integer root_part_0)) [ \n\t]* "}" """, [{"foo": 1, "bar": 2}, {"foo": 1, "bar": 2, "baz": 3}], [{"foo": 1}], ), ] @pytest.mark.parametrize( "schema, expected_grammar, accepted_instances, rejected_instances", schema__grammar__accepted_instances__rejected_instances__test_non_strict, ) def test_non_strict( schema: Dict[str, Any], expected_grammar: str, accepted_instances: List[Any], rejected_instances: List[Any], ): check_schema_with_grammar(schema, expected_grammar, strict_mode=False) for instance in accepted_instances: check_schema_with_instance(schema, instance, is_accepted=True, strict_mode=False) for instance in rejected_instances: check_schema_with_instance(schema, instance, is_accepted=False, strict_mode=False) def test_enum_const(): class Field(Enum): FOO = "foo" BAR = "bar" class MainModel(BaseModel): bars: Literal["a"] str_values: Literal['a\n\r"'] foo: Literal["a", "b", "c"] values: Literal[1, "a", True] field: Field ebnf_grammar = basic_json_rules_ebnf_no_space + ( r"""root_prop_0 ::= "\"a\"" root_prop_1 ::= "\"a\\n\\r\\\"\"" root_prop_2 ::= ("\"a\"") | ("\"b\"") | ("\"c\"") root_prop_3 ::= ("1") | ("\"a\"") | ("true") defs_Field ::= ("\"foo\"") | ("\"bar\"") root_prop_4 ::= defs_Field root_part_3 ::= ", " "\"field\"" ": " root_prop_4 "" root_part_2 ::= ", " "\"values\"" ": " root_prop_3 root_part_3 root_part_1 ::= ", " "\"foo\"" ": " root_prop_2 root_part_2 root_part_0 ::= ", " "\"str_values\"" ": " root_prop_1 root_part_1 root ::= "{" "" (("\"bars\"" ": " root_prop_0 root_part_0)) "" "}" """ ) schema = MainModel.model_json_schema() instance = MainModel(foo="a", values=1, bars="a", str_values='a\n\r"', field=Field.FOO) check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=False) check_schema_with_instance(schema, instance, any_whitespace=False) def test_empty_enum_rejected(): """Empty enum [] should raise error, not produce invalid grammar.""" schema_obj = '{"type":"object","properties":{"x":{"type":"string","enum":[]}},"required":["x"]}' with pytest.raises(RuntimeError): xgr.Grammar.from_json_schema(schema_obj) schema_str = '{"type":"string","enum":[]}' with pytest.raises(RuntimeError): xgr.Grammar.from_json_schema(schema_str) schema_int = '{"type":"integer","enum":[]}' with pytest.raises(RuntimeError): xgr.Grammar.from_json_schema(schema_int) def test_optional(): class MainModel(BaseModel): num: int = 0 opt_bool: Optional[bool] = None size: Optional[float] name: str = "" ebnf_grammar = basic_json_rules_ebnf_no_space + ( r"""root_prop_1 ::= basic_boolean | basic_null root_prop_2 ::= basic_number | basic_null root_part_2 ::= "" | ", " "\"name\"" ": " basic_string "" root_part_1 ::= ", " "\"size\"" ": " root_prop_2 root_part_2 root_part_0 ::= root_part_1 | ", " "\"opt_bool\"" ": " root_prop_1 root_part_1 root ::= "{" "" (("\"num\"" ": " basic_integer root_part_0) | ("\"opt_bool\"" ": " root_prop_1 root_part_1) | ("\"size\"" ": " root_prop_2 root_part_2)) "" "}" """ ) schema = MainModel.model_json_schema() check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=False) instance = MainModel(num=42, opt_bool=True, size=3.14, name="foo") check_schema_with_instance(schema, instance, any_whitespace=False) instance = MainModel(size=None) check_schema_with_instance(schema, instance, any_whitespace=False) check_schema_with_instance(schema, '{"size": null}', any_whitespace=False) check_schema_with_instance(schema, '{"size": null, "name": "foo"}', any_whitespace=False) check_schema_with_instance( schema, '{"num": 1, "size": null, "name": "foo"}', any_whitespace=False ) def test_all_optional(): class MainModel(BaseModel): size: int = 0 state: bool = False num: float = 0 ebnf_grammar = basic_json_rules_ebnf_no_space + ( r"""root_part_1 ::= "" | ", " "\"num\"" ": " basic_number "" root_part_0 ::= root_part_1 | ", " "\"state\"" ": " basic_boolean root_part_1 root ::= ("{" "" (("\"size\"" ": " basic_integer root_part_0) | ("\"state\"" ": " basic_boolean root_part_1) | ("\"num\"" ": " basic_number "")) "" "}") | "{" "}" """ ) schema = MainModel.model_json_schema() check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=False) instance = MainModel(size=42, state=True, num=3.14) check_schema_with_instance(schema, instance, any_whitespace=False) check_schema_with_instance(schema, '{"state": false}', any_whitespace=False) check_schema_with_instance(schema, '{"size": 1, "num": 1.5}', any_whitespace=False) def test_all_optional_non_strict(): class MainModel(BaseModel): size: int = 0 state: bool = False num: float = 0 ebnf_grammar_non_strict = basic_json_rules_ebnf_no_space + ( r"""root_addl ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object root_addl_key ::= ["] (("\"" | [^ns\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "n" ("\"" | [^u\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "u" ("\"" | [^m\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "m" ([^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub))) | "s" ("\"" | [^it\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "i" ("\"" | [^z\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "z" ("\"" | [^e\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "e" ([^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub))) | "t" ("\"" | [^a\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "a" ("\"" | [^t\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "t" ("\"" | [^e\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "e" ([^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub))))))) (= [ \n\t]* [,}\]:]) root_part_2 ::= (", " root_addl_key ": " root_addl)* root_part_1 ::= root_part_2 | ", " "\"num\"" ": " basic_number root_part_2 root_part_0 ::= root_part_1 | ", " "\"state\"" ": " basic_boolean root_part_1 root ::= ("{" "" (("\"size\"" ": " basic_integer root_part_0) | ("\"state\"" ": " basic_boolean root_part_1) | ("\"num\"" ": " basic_number root_part_2) | root_addl_key ": " root_addl root_part_2) "" "}") | "{" "}" """ ) schema = MainModel.model_json_schema() check_schema_with_grammar( schema, ebnf_grammar_non_strict, any_whitespace=False, strict_mode=False ) check_schema_with_instance( schema, '{"size": 1, "num": 1.5, "other": false}', any_whitespace=False, strict_mode=False ) check_schema_with_instance(schema, '{"other": false}', any_whitespace=False, strict_mode=False) def test_empty(): class MainModel(BaseModel): pass ebnf_grammar = basic_json_rules_ebnf_no_space + ( r"""root ::= ("{" "}") | "{" "}" """ ) schema = MainModel.model_json_schema() check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=False) instance = MainModel() check_schema_with_instance(schema, instance, any_whitespace=False) check_schema_with_instance(schema, '{"tmp": 123}', any_whitespace=False, strict_mode=False) def test_reference(): class Foo(BaseModel): count: int size: Optional[float] = None class Bar(BaseModel): apple: str = "x" banana: str = "y" class MainModel(BaseModel): foo: Foo bars: List[Bar] instance = MainModel( foo=Foo(count=42, size=3.14), bars=[Bar(apple="a", banana="b"), Bar(apple="c", banana="d")] ) ebnf_grammar = basic_json_rules_ebnf_no_space + ( r"""defs_Foo_prop_1 ::= basic_number | basic_null defs_Foo_part_0 ::= "" | ", " "\"size\"" ": " defs_Foo_prop_1 "" defs_Foo ::= "{" "" (("\"count\"" ": " basic_integer defs_Foo_part_0)) "" "}" root_prop_0 ::= defs_Foo defs_Bar_part_0 ::= "" | ", " "\"banana\"" ": " basic_string "" defs_Bar ::= ("{" "" (("\"apple\"" ": " basic_string defs_Bar_part_0) | ("\"banana\"" ": " basic_string "")) "" "}") | "{" "}" root_prop_1_additional ::= defs_Bar root_prop_1 ::= (("[" "" root_prop_1_additional (", " root_prop_1_additional)* "" "]") | ("[" "" "]")) root_part_0 ::= ", " "\"bars\"" ": " root_prop_1 "" root ::= "{" "" (("\"foo\"" ": " root_prop_0 root_part_0)) "" "}" """ ) schema = MainModel.model_json_schema() check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=False) check_schema_with_instance(schema, instance, any_whitespace=False) def test_reference_schema(): # Test simple reference with $defs schema = { "type": "object", "properties": {"value": {"$ref": "#/$defs/nested"}}, "required": ["value"], "$defs": { "nested": { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], } }, } instance = {"value": {"name": "John", "age": 30}} instance_rejected = {"value": {"name": "John"}} check_schema_with_instance(schema, instance, any_whitespace=False) check_schema_with_instance(schema, instance_rejected, is_accepted=False, any_whitespace=False) # Test simple reference with definitions schema_def = { "type": "object", "properties": {"value": {"$ref": "#/definitions/nested"}}, "required": ["value"], "definitions": { "nested": { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], } }, } check_schema_with_instance(schema_def, instance, any_whitespace=False) check_schema_with_instance( schema_def, instance_rejected, is_accepted=False, any_whitespace=False ) # Test multi-level reference path schema_multi = { "type": "object", "properties": {"value": {"$ref": "#/$defs/level1/level2/nested"}}, "required": ["value"], "$defs": { "level1": { "level2": { "nested": { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], } } } }, } check_schema_with_instance(schema_multi, instance, any_whitespace=False) check_schema_with_instance( schema_multi, instance_rejected, is_accepted=False, any_whitespace=False ) # Test nested reference schema_nested = { "type": "object", "properties": {"value": {"$ref": "#/definitions/node_a"}}, "required": ["value"], "definitions": { "node_a": { "type": "object", "properties": { "name": {"type": "string"}, "child": {"$ref": "#/definitions/node_b"}, }, "required": ["name"], }, "node_b": { "type": "object", "properties": {"id": {"type": "integer"}}, "required": ["id"], }, }, } instance_nested = {"value": {"name": "first", "child": {"id": 1}}} instance_nested_rejected = {"value": {"name": "first", "child": {}}} check_schema_with_instance(schema_nested, instance_nested, any_whitespace=False) check_schema_with_instance( schema_nested, instance_nested_rejected, is_accepted=False, any_whitespace=False ) # Test schema with self-recursion through $defs schema_self_recursive = { "type": "object", "properties": {"value": {"$ref": "#/$defs/node"}}, "required": ["value"], "$defs": { "node": { "type": "object", "properties": {"id": {"type": "integer"}, "next": {"$ref": "#/$defs/node"}}, "required": ["id"], } }, } instance_self_recursive = {"value": {"id": 1, "next": {"id": 2, "next": {"id": 3}}}} instance_self_recursive_1 = {"value": {"id": 1}} instance_self_recursive_rejected = {"value": {"id": 1, "next": {"next": {"id": 3}}}} check_schema_with_instance(schema_self_recursive, instance_self_recursive, any_whitespace=False) check_schema_with_instance( schema_self_recursive, instance_self_recursive_1, any_whitespace=False ) check_schema_with_instance( schema_self_recursive, instance_self_recursive_rejected, is_accepted=False, any_whitespace=False, ) # Test schema with circular references between multiple schemas schema_circular = { "type": "object", "properties": {"value": {"$ref": "#/$defs/schema_a"}}, "required": ["value"], "$defs": { "schema_a": { "type": "object", "properties": {"name": {"type": "string"}, "next": {"$ref": "#/$defs/schema_b"}}, "required": ["name", "next"], }, "schema_b": { "type": "object", "properties": {"id": {"type": "integer"}, "child": {"$ref": "#/$defs/schema_a"}}, "required": ["id"], }, }, } instance_circular = { "value": { "name": "first", "next": {"id": 1, "child": {"name": "second", "next": {"id": 2}}}, } } instance_circular_complex = { # fmt: off "value": { "name": "root", "next": { "id": 1, "child": { "name": "level1", "next": { "id": 2, "child": { "name": "level2", "next": { "id": 3, "child": { "name": "level3", "next": { "id": 4, "child": {"name": "level4", "next": {"id": 5}}, }, }, }, }, }, }, }, } # fmt: on } instance_circular_rejected = { "value": {"name": "first", "next": {"child": {"name": "second", "next": {"id": 2}}}} } check_schema_with_instance(schema_circular, instance_circular, any_whitespace=False) check_schema_with_instance(schema_circular, instance_circular_complex, any_whitespace=False) check_schema_with_instance( schema_circular, instance_circular_rejected, is_accepted=False, any_whitespace=False ) # Test self-referential schema schema_recursive = { "type": "object", "properties": { "name": {"type": "string"}, "children": {"type": "array", "items": {"$ref": "#"}}, }, "required": ["name"], } instance_recursive = { "name": "root", "children": [{"name": "child1", "children": [{"name": "grandchild1"}]}, {"name": "child2"}], } instance_recursive_rejected = {"children": [{"name": "child1"}]} check_schema_with_instance(schema_recursive, instance_recursive, any_whitespace=False) check_schema_with_instance( schema_recursive, instance_recursive_rejected, is_accepted=False, any_whitespace=False ) def test_union(): class Cat(BaseModel): name: str color: str class Dog(BaseModel): name: str breed: str ta = TypeAdapter(Union[Cat, Dog]) model_schema = ta.json_schema() ebnf_grammar = basic_json_rules_ebnf_no_space + ( r"""defs_Cat_part_0 ::= ", " "\"color\"" ": " basic_string "" defs_Cat ::= "{" "" (("\"name\"" ": " basic_string defs_Cat_part_0)) "" "}" root_case_0 ::= defs_Cat defs_Dog_part_0 ::= ", " "\"breed\"" ": " basic_string "" defs_Dog ::= "{" "" (("\"name\"" ": " basic_string defs_Dog_part_0)) "" "}" root_case_1 ::= defs_Dog root ::= root_case_0 | root_case_1 """ ) check_schema_with_grammar(model_schema, ebnf_grammar, any_whitespace=False) check_schema_with_instance(model_schema, Cat(name="kitty", color="black"), any_whitespace=False) check_schema_with_instance( model_schema, Dog(name="doggy", breed="bulldog"), any_whitespace=False ) check_schema_with_instance( model_schema, '{"name": "kitty", "test": "black"}', False, any_whitespace=False ) def test_anyof_oneof(): schema = { "type": "object", "properties": {"name": {"anyOf": [{"type": "string"}, {"type": "integer"}]}}, } schema_accepted_1 = '{"name": "John"}' schema_accepted_2 = '{"name": 123}' schema_rejected = '{"name": {"a": 1}}' check_schema_with_instance(schema, schema_accepted_1, any_whitespace=False) check_schema_with_instance(schema, schema_accepted_2, any_whitespace=False) check_schema_with_instance(schema, schema_rejected, is_accepted=False, any_whitespace=False) schema = { "type": "object", "properties": {"name": {"oneOf": [{"type": "string"}, {"type": "integer"}]}}, } schema_accepted_1 = '{"name": "John"}' schema_accepted_2 = '{"name": 123}' schema_rejected = '{"name": {"a": 1}}' check_schema_with_instance(schema, schema_accepted_1, any_whitespace=False) check_schema_with_instance(schema, schema_accepted_2, any_whitespace=False) check_schema_with_instance(schema, schema_rejected, is_accepted=False, any_whitespace=False) def test_alias(): class MainModel(BaseModel): test: str = Field(..., alias="name") ebnf_grammar = basic_json_rules_ebnf_no_space + ( r"""root ::= "{" "" (("\"name\"" ": " basic_string "")) "" "}" """ ) check_schema_with_grammar(MainModel.model_json_schema(), ebnf_grammar, any_whitespace=False) instance = MainModel(name="kitty") instance_str = json.dumps(instance.model_dump(mode="json", round_trip=True, by_alias=False)) check_schema_with_instance( MainModel.model_json_schema(by_alias=False), instance_str, any_whitespace=False ) instance_str = json.dumps(instance.model_dump(mode="json", round_trip=True, by_alias=True)) check_schema_with_instance( MainModel.model_json_schema(by_alias=True), instance_str, any_whitespace=False ) # property name contains space class MainModelSpace(BaseModel): test: Literal["abc"] = Field(..., alias="name 1") ebnf_grammar_space = basic_json_rules_ebnf_no_space + ( r"""root_prop_0 ::= "\"abc\"" root ::= "{" "" (("\"name 1\"" ": " root_prop_0 "")) "" "}" """ ) check_schema_with_grammar( MainModelSpace.model_json_schema(), ebnf_grammar_space, any_whitespace=False ) instance_space = MainModelSpace(**{"name 1": "abc"}) instance_space_str = json.dumps( instance_space.model_dump(mode="json", round_trip=True, by_alias=True) ) check_schema_with_instance( MainModelSpace.model_json_schema(by_alias=True), instance_space_str, any_whitespace=False ) def test_restricted_string(): class MainModel(BaseModel): restricted_string: str = Field(..., pattern=r"[a-f]") instance = MainModel(restricted_string="a") instance_str = json.dumps(instance.model_dump(mode="json")) check_schema_with_instance(MainModel.model_json_schema(), instance_str, any_whitespace=False) check_schema_with_instance( MainModel.model_json_schema(), '{"restricted_string": "j"}', is_accepted=False, any_whitespace=False, ) def test_complex_restrictions(): class RestrictedModel(BaseModel): restricted_string: str = Field(..., pattern=r"[^\"]*") restricted_value: int = Field(..., strict=True, ge=0, lt=44) # working instance instance = RestrictedModel(restricted_string="abd", restricted_value=42) instance_str = json.dumps(instance.model_dump(mode="json")) check_schema_with_instance( RestrictedModel.model_json_schema(), instance_str, any_whitespace=False ) instance_err = RestrictedModel(restricted_string='"', restricted_value=42) instance_str = json.dumps(instance_err.model_dump(mode="json")) check_schema_with_instance( RestrictedModel.model_json_schema(), instance_str, is_accepted=False, any_whitespace=False ) check_schema_with_instance( RestrictedModel.model_json_schema(), '{"restricted_string": "j", "restricted_value": 45}', is_accepted=False, any_whitespace=False, ) def test_dynamic_model(): class MainModel(BaseModel): restricted_string: str = Field(..., pattern=r"[a-f]") additional_fields = {"restricted_string_dynamic": (str, Field(..., pattern=r"[a-x]"))} CompleteModel: Type[BaseModel] = create_model( "CompleteModel", __base__=MainModel, **additional_fields ) instance = CompleteModel(restricted_string="a", restricted_string_dynamic="j") instance_str = json.dumps(instance.model_dump(mode="json")) check_schema_with_instance( CompleteModel.model_json_schema(), instance_str, any_whitespace=False ) def test_any_whitespace(): class SimpleModel(BaseModel): value: str arr: List[int] obj: Dict[str, int] schema = SimpleModel.model_json_schema() ebnf_grammar = basic_json_rules_ebnf + ( r"""root_prop_1 ::= (("[" [ \n\t]* basic_integer ([ \n\t]* "," [ \n\t]* basic_integer)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) root_prop_2 ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_integer ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_integer)* [ \n\t]* "}") | "{" [ \n\t]* "}" root_part_1 ::= [ \n\t]* "," [ \n\t]* "\"obj\"" [ \n\t]* ":" [ \n\t]* root_prop_2 "" root_part_0 ::= [ \n\t]* "," [ \n\t]* "\"arr\"" [ \n\t]* ":" [ \n\t]* root_prop_1 root_part_1 root ::= "{" [ \n\t]* (("\"value\"" [ \n\t]* ":" [ \n\t]* basic_string root_part_0)) [ \n\t]* "}" """ ) check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=True, strict_mode=True) ebnf_grammar = basic_json_rules_ebnf + ( r"""root_prop_1 ::= (("[" [ \n\t]* basic_integer ([ \n\t]* "," [ \n\t]* basic_integer)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) root_prop_2 ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_integer ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_integer)* [ \n\t]* "}") | "{" [ \n\t]* "}" root_addl ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object root_addl_key ::= ["] (("\"" | [^aov\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "a" ("\"" | [^r\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "r" ("\"" | [^r\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "r" ([^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub))) | "o" ("\"" | [^b\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "b" ("\"" | [^j\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "j" ([^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub))) | "v" ("\"" | [^a\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "a" ("\"" | [^l\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "l" ("\"" | [^u\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "u" ("\"" | [^e\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub | "e" ([^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub))))))) (= [ \n\t]* [,}\]:]) root_part_2 ::= ([ \n\t]* "," [ \n\t]* root_addl_key [ \n\t]* ":" [ \n\t]* root_addl)* root_part_1 ::= [ \n\t]* "," [ \n\t]* "\"obj\"" [ \n\t]* ":" [ \n\t]* root_prop_2 root_part_2 root_part_0 ::= [ \n\t]* "," [ \n\t]* "\"arr\"" [ \n\t]* ":" [ \n\t]* root_prop_1 root_part_1 root ::= "{" [ \n\t]* (("\"value\"" [ \n\t]* ":" [ \n\t]* basic_string root_part_0)) [ \n\t]* "}" """ ) check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=True, strict_mode=False) # Test that different whitespace variations are accepted when any_whitespace=True instances = [ '{"value": "test", "arr": [1, 2], "obj": {"a": 1}}', '{ "value" : "test", "arr": [1, 2], "obj": {"a": 1} }', '{\n "value" : "test",\n "arr" : [1, 2],\n "obj" : {"a": 1}\n}', '{\t"value"\t:\t"test",\t"arr":\t[1,\t2],\t"obj":\t{"a":\t1}\t}', ] for instance in instances: check_schema_with_instance(schema, instance, any_whitespace=True) schema__err_message__test_array_schema_error_cases = [ ({"type": "array", "prefixItems": {"type": "string"}}, "prefixItems must be an array"), ( {"type": "array", "prefixItems": ["not an object"]}, "prefixItems must be an array of objects or booleans", ), ({"type": "array", "prefixItems": [False]}, "prefixItems contains false"), ({"type": "array", "items": "not an object"}, "items must be a boolean or an object"), ( {"type": "array", "unevaluatedItems": "not an object"}, "unevaluatedItems must be a boolean or an object", ), ({"type": "array", "minItems": "not an integer"}, "minItems must be an integer"), ({"type": "array", "maxItems": -1}, "maxItems must be a non-negative integer"), ({"type": "array", "minItems": 5, "maxItems": 3}, "minItems is greater than maxItems: 5 > 3"), ( {"type": "array", "prefixItems": [{}, {}, {}], "maxItems": 2}, "maxItems is less than the number of prefixItems: 2 < 3", ), ( {"type": "array", "prefixItems": [{}, {}], "minItems": 3, "items": False}, "minItems is greater than the number of prefixItems, but additional items are not " "allowed: 3 > 2", ), ] @pytest.mark.parametrize("schema, err_message", schema__err_message__test_array_schema_error_cases) def test_array_schema_error_cases(schema: Dict[str, Any], err_message: str): with pytest.raises(Exception) as e: _json_schema_to_ebnf(schema) assert err_message in str(e.value) schema__expected_grammar__instances__test_array_schema = [ ( { "type": "array", "items": { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], }, }, ( basic_json_rules_ebnf + r"""root_additional_part_0 ::= [ \n\t]* "," [ \n\t]* "\"age\"" [ \n\t]* ":" [ \n\t]* basic_integer "" root_additional ::= "{" [ \n\t]* (("\"name\"" [ \n\t]* ":" [ \n\t]* basic_string root_additional_part_0)) [ \n\t]* "}" root ::= (("[" [ \n\t]* root_additional ([ \n\t]* "," [ \n\t]* root_additional)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) """ ), [ ([{"name": "John", "age": 30}, {"name": "Jane", "age": 25}], True), ([{"name": "John"}], False), ], ), ( { "type": "array", "prefixItems": [ { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], }, {"type": "integer"}, {"type": "string"}, ], "additionalItems": False, }, ( basic_json_rules_ebnf + r"""root_item_0_part_0 ::= [ \n\t]* "," [ \n\t]* "\"age\"" [ \n\t]* ":" [ \n\t]* basic_integer "" root_item_0 ::= "{" [ \n\t]* (("\"name\"" [ \n\t]* ":" [ \n\t]* basic_string root_item_0_part_0)) [ \n\t]* "}" root ::= ("[" [ \n\t]* (root_item_0 [ \n\t]* "," [ \n\t]* basic_integer [ \n\t]* "," [ \n\t]* basic_string) [ \n\t]* "]") """ ), [ ([{"name": "John", "age": 30}, 42, "test"], True), ([{"name": "John", "age": 30}, 42], False), ([{"name": "John", "age": 30}, "test", 42], False), ([{"name": "John"}, 42, "test"], False), ], ), ( { "type": "array", "prefixItems": [ { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], }, {"type": "integer"}, ], "unevaluatedItems": { "type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"], }, }, ( basic_json_rules_ebnf + r"""root_item_0_part_0 ::= [ \n\t]* "," [ \n\t]* "\"age\"" [ \n\t]* ":" [ \n\t]* basic_integer "" root_item_0 ::= "{" [ \n\t]* (("\"name\"" [ \n\t]* ":" [ \n\t]* basic_string root_item_0_part_0)) [ \n\t]* "}" root_additional ::= "{" [ \n\t]* (("\"name\"" [ \n\t]* ":" [ \n\t]* basic_string "")) [ \n\t]* "}" root ::= ("[" [ \n\t]* (root_item_0 [ \n\t]* "," [ \n\t]* basic_integer) ([ \n\t]* "," [ \n\t]* root_additional)* [ \n\t]* "]") """ ), [ ([{"name": "John", "age": 30}, 42, {"name": "Jane"}], True), ([{"name": "John", "age": 30}, 42], True), ([{"name": "John", "age": 30}, 42, 123], False), ([{"name": "John", "age": 30}, {"name": "Jane"}], False), ], ), ] @pytest.mark.parametrize( "schema, expected_grammar, instances", schema__expected_grammar__instances__test_array_schema ) def test_array_schema( schema: Dict[str, Any], expected_grammar: str, instances: List[Tuple[Any, bool]] ): check_schema_with_grammar(schema, expected_grammar) for instance, is_accepted in instances: check_schema_with_instance(schema, instance, is_accepted=is_accepted) schema__expected_grammar__instances__test_array_schema_min_max = [ # prefix empty, additional items not allowed ( {"type": "array", "items": False, "prefixItems": []}, ( basic_json_rules_ebnf + r"""root ::= ("[" [ \n\t]* "]") """ ), [([], True), ([1], False), ([1, 2], False)], ), # prefix empty, additional items allowed, min=0 max=0 ( {"type": "array", "items": {"type": "integer"}, "minItems": 0, "maxItems": 0}, ( basic_json_rules_ebnf + r"""root ::= ("[" [ \n\t]* "]") """ ), [([], True), ([1], False), ([1, 2], False)], ), # prefix empty, additional items allowed, min=0 max>0 ( {"type": "array", "items": {"type": "integer"}, "minItems": 0, "maxItems": 2}, ( basic_json_rules_ebnf + r"""root ::= (("[" [ \n\t]* basic_integer ([ \n\t]* "," [ \n\t]* basic_integer)? [ \n\t]* "]") | ("[" [ \n\t]* "]")) """ ), [([], True), ([1], True), ([1, 2], True), ([1, 2, 3], False)], ), # prefix empty, additional items allowed, min>0 ( {"type": "array", "items": {"type": "integer"}, "minItems": 2, "maxItems": 3}, ( basic_json_rules_ebnf + r"""root ::= ("[" [ \n\t]* basic_integer ([ \n\t]* "," [ \n\t]* basic_integer){1,2} [ \n\t]* "]") """ ), [([], False), ([1], False), ([1, 2], True), ([1, 2, 3], True), ([1, 2, 3, 4], False)], ), # prefix non-empty, additional items not allowed ( {"type": "array", "items": False, "prefixItems": [{"type": "string"}, {"type": "integer"}]}, ( basic_json_rules_ebnf + r"""root ::= ("[" [ \n\t]* (basic_string [ \n\t]* "," [ \n\t]* basic_integer) [ \n\t]* "]") """ ), [(["foo", 42], True), (["foo", 42, "bar"], False), (["foo"], False), ([42, "foo"], False)], ), # prefix non-empty, additional items allowed ( { "type": "array", "prefixItems": [{"type": "string"}, {"type": "integer"}], "items": {"type": "boolean"}, "minItems": 3, "maxItems": 4, }, ( basic_json_rules_ebnf + r"""root ::= ("[" [ \n\t]* (basic_string [ \n\t]* "," [ \n\t]* basic_integer) ([ \n\t]* "," [ \n\t]* basic_boolean){1,2} [ \n\t]* "]") """ ), [ (["foo", 42, True], True), (["foo", 42, True, False], True), (["foo", 42], False), (["foo", 42, True, False, True], False), (["foo", 42, "bar"], False), ], ), # prefix non-empty, additional items allowed, maxItems not set ( { "type": "array", "prefixItems": [{"type": "string"}, {"type": "integer"}], "items": {"type": "boolean"}, "minItems": 3, }, ( basic_json_rules_ebnf + r"""root ::= ("[" [ \n\t]* (basic_string [ \n\t]* "," [ \n\t]* basic_integer) ([ \n\t]* "," [ \n\t]* basic_boolean)+ [ \n\t]* "]") """ ), [ (["foo", 42, True], True), (["foo", 42, True, False], True), (["foo", 42, True, False, True], True), (["foo", 42], False), (["foo", 42, "bar"], False), ], ), ] @pytest.mark.parametrize( "schema, expected_grammar, instances", schema__expected_grammar__instances__test_array_schema_min_max, ) def test_array_schema_min_max( schema: Dict[str, Any], expected_grammar: str, instances: List[Tuple[Any, bool]] ): grammar_ebnf = _json_schema_to_ebnf(schema) assert grammar_ebnf == expected_grammar for instance, is_accepted in instances: check_schema_with_instance(schema, instance, is_accepted=is_accepted) def test_array_with_only_items_keyword(): schema = { "items": { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], } } instance_accepted = [{"name": "John", "age": 30}, {"name": "Jane", "age": 25}] instance_rejected = [{"name": "John"}] check_schema_with_instance(schema, instance_accepted, any_whitespace=False) check_schema_with_instance(schema, instance_rejected, is_accepted=False, any_whitespace=False) schema_prefix_items = { "prefixItems": [ { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], }, { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], }, ] } check_schema_with_instance(schema_prefix_items, instance_accepted, any_whitespace=False) check_schema_with_instance( schema_prefix_items, instance_rejected, is_accepted=False, any_whitespace=False ) schema_unevaluated_items = { "unevaluatedItems": { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], } } check_schema_with_instance(schema_unevaluated_items, instance_accepted, any_whitespace=False) check_schema_with_instance( schema_unevaluated_items, instance_rejected, is_accepted=False, any_whitespace=False ) def test_object_with_only_properties_keyword(): schema = { "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], } instance_accepted = {"name": "John", "age": 30} instance_rejected = {"name": "John"} check_schema_with_instance(schema, instance_accepted, any_whitespace=False) check_schema_with_instance(schema, instance_rejected, is_accepted=False, any_whitespace=False) schema_additional_properties = {"additionalProperties": {"type": "string"}} instance_accepted = {"name": "John"} instance_rejected = {"name": "John", "age": 30} check_schema_with_instance( schema_additional_properties, instance_accepted, any_whitespace=False ) check_schema_with_instance( schema_additional_properties, instance_rejected, is_accepted=False, any_whitespace=False ) schema_unevaluated_properties = {"unevaluatedProperties": {"type": "string"}} check_schema_with_instance( schema_unevaluated_properties, instance_accepted, any_whitespace=False ) check_schema_with_instance( schema_unevaluated_properties, instance_rejected, is_accepted=False, any_whitespace=False ) def test_object_with_pattern_properties_and_property_names(): schema = { "type": "object", "patternProperties": { "^[a-zA-Z]+$": {"type": "string"}, "^[0-9]+$": {"type": "integer"}, "^[a-zA-Z]*_[0-9]*$": {"type": "object"}, }, } instance_accepted = [ {"aBcDe": "aaa"}, {"12345": 12345}, {"abc_123": {"key": "value"}}, {"_": {"key": "value"}}, {"a": "value", "b": "another_value", "000": 12345, "abc_123": {"key": "value"}}, {"000": 12345, "a": "value", "abc_123": {"key": "value"}, "b": "another_value"}, ] instance_rejected = [ {"233A": "adfa"}, {"aBcDe": 12345}, {"12345": "aaa"}, {"abc_123": 12345}, {"a": "value", "b": "another_value", "000": 12345, "abc_123": "aaa"}, { "a": "value", "b": "another_value", "000": 12345, "???": {"key": "value"}, "abc_123": {"key": "value"}, }, {"000": 12345, "a": "value", "abc_123": {"key": "value"}, "b": 12345}, ] for instance in instance_accepted: check_schema_with_instance(schema, instance, any_whitespace=False) for instance in instance_rejected: check_schema_with_instance(schema, instance, is_accepted=False, any_whitespace=False) schema = {"type": "object", "propertyNames": {"pattern": "^[a-zA-Z0-9_]+$"}} instance_accepted = [ {"aBcDe": "aaa"}, {"12345": 12345}, {"abc_123": {"key": "value"}}, {"_": {"key": "value"}}, {"a": "value", "b": "another_value", "000": 12345, "abc_123": {"key": "value"}}, {"000": 12345, "a": "value", "abc_123": {"key": "value"}, "b": "another_value"}, ] instance_rejected = [ {"aBc?De": "aaa"}, {"1234|5": 12345}, {"abc_1.23": {"key": "value"}}, {"_/": {"key": "value"}}, {"a&": "value", "b": "another_value", "000": 12345, "abc_123": {"key": "value"}}, {"00(0": 12345, "a": "value", "abc_123": {"key": "value"}, "b": "another_value"}, ] for instance in instance_accepted: check_schema_with_instance(schema, instance, any_whitespace=False) for instance in instance_rejected: check_schema_with_instance(schema, instance, is_accepted=False, any_whitespace=False) def test_object_with_property_numbers(): base_schema = { "properties": { "key1": {"type": "string"}, "key2": {"type": "string"}, "key3": {"type": "string"}, "key4": {"type": "string"}, } } # fmt: off instances = [ ({}, 0 , False), ({"key1": "value1"}, 1, False), ({"key4": "value4"}, 1, False), ({"additional_key": "value"}, 1, True), ({"key1": "value1", "additional_key": "value"}, 2, True), ({"key1": "value1", "key2": "value2"}, 2, False), ({"key1": "value1", "key4": "value4"}, 2, False), ({"key2": "value2", "key3": "value3"}, 2, False), ({"key1": "value1", "key2": "value2", "additional_key": "value"}, 3, True), ({"key1": "value1", "key4": "value4", "additional_key": "value"}, 3, True), ({"key3": "value3", "key4": "value4", "additional_key": "value"}, 3, True), ({"key1": "value1", "key2": "value2", "key3": "value3"}, 3, False), ({"key2": "value2", "key3": "value3", "key4": "value4"}, 3, False), ({"key1": "value1", "key2": "value2", "key3": "value3", "additional_key": "value"}, 4, True), ({"key2": "value2", "key3": "value3", "key4": "value4", "additional_key": "value"}, 4, True), ({"key1": "value1", "key2": "value2", "key3": "value3", "key4": "value4"}, 4, False), ({"key1": "value1", "key2": "value2", "key3": "value3", "key4": "value4", "additional_key": "value"}, 5, True), ({"additional_key1": "value1", "additional_key2": "value2"}, 2, True), ({"key1": "value1", "key2": "value2", "additional_key1": "value", "additional_key2": "value2"}, 4 , True), ({"key1": "value1", "key2": "value2", "key3": "value3", "additional_key1": "value", "additional_key2": "value2"}, 5, True), ({"key1": "value1", "key2": "value2", "key3": "value3", "key4": "value4", "additional_key1": "value", "additional_key2": "value2"}, 6, True), ({"additional_key1": "value1", "additional_key2": "value2", "additional_key3": "value3"}, 3, True), ({"additional_key1": "value1", "additional_key2": "value2", "additional_key3": "value3", "additional_key4": "value4", "additional_key5": "value5", "additional_key6": "value6"}, 6, True), ] # fmt: on # Case 1. all properties are optional # Case 1.1. no additional properties schema = {**base_schema} for instance, num_properties, have_additional in instances: check_schema_with_instance( schema, instance, is_accepted=not have_additional, any_whitespace=False ) for lower_bound in range(8): schema = {**base_schema, "minProperties": lower_bound} for instance, num_properties, have_additional in instances: if lower_bound > len(base_schema["properties"]): continue if num_properties < lower_bound: is_accepted = False else: is_accepted = not have_additional check_schema_with_instance( schema, instance, is_accepted=is_accepted, any_whitespace=False ) for upper_bound in range(lower_bound, 8): schema = {**base_schema, "minProperties": lower_bound, "maxProperties": upper_bound} for instance, num_properties, have_additional in instances: if lower_bound > len(base_schema["properties"]): continue if num_properties < lower_bound or num_properties > upper_bound: is_accepted = False else: is_accepted = not have_additional check_schema_with_instance( schema, instance, is_accepted=is_accepted, any_whitespace=False ) # Case 1.2. additional properties allowed schema = {**base_schema, "additionalProperties": {"type": "string"}} for instance, num_properties, have_additional in instances: check_schema_with_instance(schema, instance, any_whitespace=False) for lower_bound in range(8): schema = { **base_schema, "minProperties": lower_bound, "additionalProperties": {"type": "string"}, } for instance, num_properties, have_additional in instances: if num_properties < lower_bound: is_accepted = False else: is_accepted = True check_schema_with_instance( schema, instance, is_accepted=is_accepted, any_whitespace=False ) for upper_bound in range(lower_bound, 8): schema = { **base_schema, "minProperties": lower_bound, "maxProperties": upper_bound, "additionalProperties": {"type": "string"}, } for instance, num_properties, have_additional in instances: if lower_bound > len(base_schema["properties"]): continue if num_properties < lower_bound or num_properties > upper_bound: is_accepted = False else: is_accepted = True check_schema_with_instance( schema, instance, is_accepted=is_accepted, any_whitespace=False ) # Case 2. required properties are defined # Case 2.1. no additional properties required_properties_instance = [ [], ["key1"], ["key3"], ["key4"], ["key1", "key2"], ["key1", "key4"], ["key3", "key4"], ["key1", "key2", "key3"], ["key2", "key3", "key4"], ["key1", "key2", "key3", "key4"], ] for required_properties in required_properties_instance: schema = {**base_schema, "required": required_properties} for instance, num_properties, have_additional in instances: is_accepted = ( all(prop in instance for prop in required_properties) and not have_additional ) check_schema_with_instance( schema, instance, is_accepted=is_accepted, any_whitespace=False ) for lower_bound in range(8): schema = {**base_schema, "minProperties": lower_bound, "required": required_properties} for instance, num_properties, have_additional in instances: if lower_bound > len(base_schema["properties"]): continue if num_properties < lower_bound: is_accepted = False else: is_accepted = ( all(prop in instance for prop in required_properties) and not have_additional ) check_schema_with_instance( schema, instance, is_accepted=is_accepted, any_whitespace=False ) for upper_bound in range(lower_bound, 8): schema = { **base_schema, "minProperties": lower_bound, "maxProperties": upper_bound, "required": required_properties, } for instance, num_properties, have_additional in instances: if lower_bound > len(base_schema["properties"]) or upper_bound < len( required_properties ): continue if num_properties < lower_bound or num_properties > upper_bound: is_accepted = False else: is_accepted = ( all(prop in instance for prop in required_properties) and not have_additional ) check_schema_with_instance( schema, instance, is_accepted=is_accepted, any_whitespace=False ) # Case 2.2. additional properties allowed for required_properties in required_properties_instance: schema = { **base_schema, "required": required_properties, "additionalProperties": {"type": "string"}, } for instance, num_properties, have_additional in instances: is_accepted = all(prop in instance for prop in required_properties) check_schema_with_instance( schema, instance, is_accepted=is_accepted, any_whitespace=False ) for lower_bound in range(8): schema = { **base_schema, "minProperties": lower_bound, "required": required_properties, "additionalProperties": {"type": "string"}, } for instance, num_properties, have_additional in instances: if lower_bound > len(base_schema["properties"]): continue if num_properties < lower_bound: is_accepted = False else: is_accepted = all(prop in instance for prop in required_properties) check_schema_with_instance( schema, instance, is_accepted=is_accepted, any_whitespace=False ) for upper_bound in range(lower_bound, 8): schema = { **base_schema, "minProperties": lower_bound, "maxProperties": upper_bound, "required": required_properties, "additionalProperties": {"type": "string"}, } for instance, num_properties, have_additional in instances: if lower_bound > len(base_schema["properties"]) or upper_bound < len( required_properties ): continue if num_properties < lower_bound or num_properties > upper_bound: is_accepted = False else: is_accepted = all(prop in instance for prop in required_properties) check_schema_with_instance( schema, instance, is_accepted=is_accepted, any_whitespace=False ) # Case 3. No properties defined for lower_bound in range(8): schema = { "type": "object", "minProperties": lower_bound, "additionalProperties": {"type": "string"}, } for instance, num_properties, have_additional in instances: if num_properties < lower_bound: is_accepted = False else: is_accepted = True check_schema_with_instance( schema, instance, is_accepted=is_accepted, any_whitespace=False ) for upper_bound in range(lower_bound, 8): schema = { "type": "object", "minProperties": lower_bound, "maxProperties": upper_bound, "additionalProperties": {"type": "string"}, } for instance, num_properties, have_additional in instances: if num_properties < lower_bound or num_properties > upper_bound: is_accepted = False else: is_accepted = True check_schema_with_instance( schema, instance, is_accepted=is_accepted, any_whitespace=False ) def test_object_error_handle(): # Test error handling for invalid object schemas def compile_from_schema(schema): xgr.Grammar.from_json_schema( json.dumps(schema), any_whitespace=True, indent=None, separators=None, strict_mode=True ) schema = {"type": "object", "properties": "not an object"} with pytest.raises(Exception) as e: compile_from_schema(schema) assert "properties must be an object" in str(e.value) schema = {"type": "object", "required": {"key": "not an array"}} with pytest.raises(Exception) as e: compile_from_schema(schema) assert "required must be an array" in str(e.value) with pytest.raises(Exception) as e: compile_from_schema({"type": "object", "patternProperties": ["not an object"]}) assert "patternProperties must be an object" in str(e.value) with pytest.raises(Exception) as e: compile_from_schema({"type": "object", "propertyNames": "not an object"}) assert "propertyNames must be an object" in str(e.value) with pytest.raises(Exception) as e: compile_from_schema({"type": "object", "propertyNames": {"type": "object"}}) assert "propertyNames must be an object that validates string" in str(e.value) with pytest.raises(Exception) as e: compile_from_schema({"type": "object", "minProperties": "not an integer"}) assert "minProperties must be an integer" in str(e.value) with pytest.raises(Exception) as e: compile_from_schema({"type": "object", "maxProperties": "not an integer"}) assert "maxProperties must be an integer" in str(e.value) with pytest.raises(Exception) as e: compile_from_schema({"type": "object", "minProperties": -1}) assert "minProperties must be a non-negative integer" in str(e.value) with pytest.raises(Exception) as e: compile_from_schema({"type": "object", "maxProperties": -1}) assert "maxProperties must be a non-negative integer" in str(e.value) with pytest.raises(Exception) as e: compile_from_schema({"type": "object", "minProperties": 5, "maxProperties": 3}) assert "minProperties is greater than maxProperties" in str(e.value) with pytest.raises(Exception) as e: compile_from_schema({"type": "object", "maxProperties": 1, "required": ["key1", "key2"]}) assert "maxProperties is less than the number of required properties" in str(e.value) with pytest.raises(Exception) as e: compile_from_schema( { "type": "object", "additionalProperties": False, "properties": {"key": {"type": "string"}}, "minProperties": 2, } ) assert ( "minProperties is greater than the number of properties, but additional properties aren't allowed" in str(e.value) ) def test_additional_properties_type_enforcement(): """Regression test for #208: additionalProperties: true must still enforce declared types for defined (non-required) properties.""" # Case 1: additionalProperties: true + empty required + wrong type -> REJECT schema = { "type": "object", "properties": {"a": {"type": "integer"}}, "additionalProperties": True, "required": [], } check_schema_with_instance( schema, '{"a": "wrong"}', is_accepted=False, any_whitespace=False, strict_mode=False ) # Case 2: Same schema + correct type -> ACCEPT check_schema_with_instance( schema, '{"a": 42}', is_accepted=True, any_whitespace=False, strict_mode=False ) # Case 3: Same schema + truly additional property (unknown key) -> ACCEPT check_schema_with_instance( schema, '{"b": "anything"}', is_accepted=True, any_whitespace=False, strict_mode=False ) # Case 4: Defined prop correct type + additional prop -> ACCEPT check_schema_with_instance( schema, '{"a": 42, "extra": "val"}', is_accepted=True, any_whitespace=False, strict_mode=False, ) # Case 5: Multiple defined properties, partial required, wrong type on non-required -> REJECT schema2 = { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "additionalProperties": True, "required": ["name"], } check_schema_with_instance( schema2, '{"name": "Alice", "age": "twenty"}', is_accepted=False, any_whitespace=False, strict_mode=False, ) # Case 6: Same schema + correct types -> ACCEPT check_schema_with_instance( schema2, '{"name": "Alice", "age": 30}', is_accepted=True, any_whitespace=False, strict_mode=False, ) # Case 7: Empty object should be accepted (no required properties in schema 1) check_schema_with_instance( schema, "{}", is_accepted=True, any_whitespace=False, strict_mode=False ) def test_generate_range_regex(): # Basic range tests assert _generate_range_regex(12, 16) == r"^((1[2-6]))$" assert _generate_range_regex(1, 10) == r"^(([1-9]|10))$" assert ( _generate_range_regex(2134, 3459) == r"^((213[4-9]|21[4-8]\d|219\d|2[2-8]\d{2}|29\d{2}|30\d{2}|3[1-3]\d{2}|34[0-5]\d))$" ) # Negative to positive range assert _generate_range_regex(-5, 10) == r"^(-([1-5])|0|([1-9]|10))$" # Pure negative range assert _generate_range_regex(-15, -10) == r"^(-(1[0-5]))$" # Large ranges assert _generate_range_regex(-1999, -100) == r"^(-([1-9]\d{2}|1\d{3}))$" assert _generate_range_regex(1, 9999) == r"^(([1-9]|[1-9]\d|[1-9]\d{2}|[1-9]\d{3}))$" # Unbounded ranges (None cases) assert _generate_range_regex(None, None) == r"^-?\d+$" assert _generate_range_regex(5, None) == r"^([5-9]|[1-9]\d{1,})$" assert _generate_range_regex(None, 0) == r"^(-[1-9]\d*|0)$" assert _generate_range_regex(-5, None) == r"^(-([1-5])|0|[1-9]\d*)$" assert _generate_range_regex(None, -2) == r"^(-[2-9]|-[1-9]\d{1,})$" # Medium range assert ( _generate_range_regex(78, 1278) == r"^((7[8-9]|8\d|9\d|[1-9]\d{2}|10\d{2}|11\d{2}|120\d|12[1-6]\d|127[0-8]))$" ) # Symmetric range around zero assert _generate_range_regex(-100, 100) == r"^(-([1-9]|[1-9]\d|100)|0|([1-9]|[1-9]\d|100))$" # Upper bound negative assert ( _generate_range_regex(None, -123) == r"^(-12[3-9]|-1[3-8]\d|-19\d|-[2-8]\d{2}|-9\d{2}|-[1-9]\d{3,})$" ) # Additional edge cases # Single number assert _generate_range_regex(5, 5) == r"^((5))$" # Zero-inclusive ranges assert _generate_range_regex(-10, 0) == r"^(-([1-9]|10)|0)$" assert _generate_range_regex(0, 10) == r"^(0|([1-9]|10))$" # Regression: multi-digit two-sided ranges must not over-accept values that # share the lower bound's leading digits (e.g. [100, 110] rejecting 111). assert _generate_range_regex(100, 110) == r"^((10\d|110))$" assert _generate_range_regex(12345, 12347) == r"^((1234[5-7]))$" # Regression: negative multi-digit maximum must cover every value below it. assert _generate_range_regex(None, -10) == r"^(-[1-9]\d|-[1-9]\d{2,})$" assert _generate_range_regex(None, -50) == r"^(-[5-9]\d|-[1-9]\d{2,})$" # Regression: positive multi-digit minimum. assert _generate_range_regex(100, None) == r"^([1-9]\d{2}|[1-9]\d{3,})$" # Bounds beyond 32 bits (the generator operates on int64). assert _generate_range_regex(10000000000, 10000000002) == r"^((1000000000[0-2]))$" instance__accepted__test_email_format = [ (r"simple@example.com", True), (r"very.common@example.com", True), (r"FirstName.LastName@EasierReading.org", True), (r"x@example.com", True), (r"long.email-address-with-hyphens@and.subdomains.example.com", True), (r"user.name+tag+sorting@example.com", True), (r"name/surname@example.com", True), (r"admin@example", True), (r"example@s.example", True), (r"\" \"@example.org", True), # (r"\"john..doe\"@example.org", True), # (r"mailhost!username@example.org", True), (r"\"very.(),:;<>[]\\\".VERY.\\\"very@\\\\ \\\"very\\\".unusual\"@strange.example.com", True), (r"user%example.com@example.org", True), (r"user-@example.org", True), (r"abc.example.com", False), (r"a@b@c@example.com", False), (r'a"b(c)d,e:f;gi[j\k]l@example.com', False), (r'just"not"right@example.com', False), (r'this is"not\allowed@example.com', False), (r"this\ still\"not\\allowed@example.com", False), (r"i.like.underscores@but_they_are_not_allowed_in_this_part", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_email_format) def test_email_format(instance: str, accepted: bool): schema = {"type": "string", "format": "email"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" ( ( [a-zA-Z0-9_!#$%&'*+/=?^`{|}~-]+ ( "." [a-zA-Z0-9_!#$%&'*+/=?^`{|}~-]+ )* ) | "\\" "\"" ( "\\" [ -~] | [ !#-[\]-~] )* "\\" "\"" ) "@" ( [A-Za-z0-9] ( [\-A-Za-z0-9]* [A-Za-z0-9] )? ) ( ( "." [A-Za-z0-9] [\-A-Za-z0-9]* [A-Za-z0-9] )* ) "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_date_format = [ (r"0000-01-01", True), (r"9999-12-31", True), (r"10-01-01", False), (r"2025-00-01", False), (r"2025-13-01", False), (r"2025-01-00", False), (r"2025-01-32", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_date_format) def test_date_format(instance: str, accepted: bool): schema = {"type": "string", "format": "date"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" ( [0-9]{4} "-" ( "0" [1-9] | "1" [0-2] ) "-" ( "0" [1-9] | [1-2] [0-9] | "3" [01] ) ) "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_time_format = [ (r"00:00:00Z", True), (r"23:59:60Z", True), (r"12:34:56Z", True), (r"12:34:56+07:08", True), (r"12:34:56-07:08", True), (r"12:34:56.7Z", True), (r"12:34:56.7+08:09", True), (r"12:34:56.7-08:09", True), (r"00:00:00", False), (r"23:59:60", False), (r"12:34:56.7", False), (r"12:34:56.7890", False), (r"24:00:00", False), (r"00:60:00", False), (r"00:00:61", False), (r"00:00:00.", False), (r"12:34:56+07:", False), (r"12:34:56-07:", False), (r"12:34:56.7+-08:09", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_time_format) def test_time_format(instance: str, accepted: bool): schema = {"type": "string", "format": "time"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] ":" ( [0-5] [0-9] | "6" "0" ) ( "." [0-9]+ )? ( "Z" | [+-] ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] ) "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_date_time_format = [ (r"2024-05-19T14:23:45Z", True), (r"2019-11-30T08:15:27+05:30", True), (r"2030-02-01T22:59:59-07:00", True), (r"2021-07-04T00:00:00.123456Z", True), (r"2022-12-31T23:45:12-03:00", True), (r"2024-12-31T23:45:60.123456Z", True), (r"2024-12-31T23:60:12.123456+05:30", False), (r"2024-13-15T14:30:00Z", False), (r"2023-02-2010:59:59Z", False), (r"2021-11-05T24:00:00+05:30", False), (r"2022-08-20T12:61:10-03:00", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_date_time_format) def test_date_time_format(instance: str, accepted: bool): schema = {"type": "string", "format": "date-time"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" ( [0-9]{4} "-" ( "0" [1-9] | "1" [0-2] ) "-" ( "0" [1-9] | [1-2] [0-9] | "3" [01] ) ) "T" ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] ":" ( [0-5] [0-9] | "6" "0" ) ( "." [0-9]+ )? ( "Z" | [+-] ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] ) "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_duration_format = [ (r"P0Y", True), (r"P12M", True), (r"P345D", True), (r"P6789W", True), (r"P01234D", True), (r"PT9H", True), (r"PT87M", True), (r"PT654S", True), (r"P1Y23M456D", True), (r"P23M456D", True), (r"P1Y0M456D", True), (r"P1Y23M", True), (r"PT9H87M654S", True), (r"PT87M654S", True), (r"PT9H0M654S", True), (r"PT9H87M", True), (r"P1Y23M456DT9H87M654S", True), (r"P", False), (r"PD", False), (r"P1", False), (r"PT", False), (r"P1Y456D", False), (r"PT9H654S", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_duration_format) def test_duration_format(instance: str, accepted: bool): schema = {"type": "string", "format": "duration"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" "P" ( ( [0-9]+ "D" | [0-9]+ "M" ( [0-9]+ "D" )? | [0-9]+ "Y" ( [0-9]+ "M" ( [0-9]+ "D" )? )? ) ( "T" ( [0-9]+ "S" | [0-9]+ "M" ( [0-9]+ "S" )? | [0-9]+ "H" ( [0-9]+ "M" ( [0-9]+ "S" )? )? ) )? | "T" ( [0-9]+ "S" | [0-9]+ "M" ( [0-9]+ "S" )? | [0-9]+ "H" ( [0-9]+ "M" ( [0-9]+ "S" )? )? ) | [0-9]+ "W" ) "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_ipv6_format = [ (r"0123:4567:890a:bced:fABC:DEF0:1234:5678", True), (r"::6666:6666:6666:6666:6666:6666", True), (r"::6666:6666:6666:6666:6666", True), (r"::6666:6666:6666:6666", True), (r"::6666:6666:6666", True), (r"::6666:6666", True), (r"::6666", True), (r"::", True), (r"8888:8888:8888:8888:8888:8888::", True), (r"8888:8888:8888:8888:8888::", True), (r"8888:8888:8888:8888::", True), (r"8888:8888:8888::", True), (r"8888:8888::", True), (r"8888::", True), (r"1111::2222", True), (r"1111:1111::2222", True), (r"1111::2222:2222", True), (r"1111:1111:1111::2222", True), (r"1111:1111::2222:2222", True), (r"1111::2222:2222:2222", True), (r"1111:1111:1111:1111::2222", True), (r"1111:1111:1111::2222:2222", True), (r"1111:1111::2222:2222:2222", True), (r"1111::2222:2222:2222:2222", True), (r"1111:1111:1111:1111:1111::2222", True), (r"1111:1111:1111:1111::2222:2222", True), (r"1111:1111:1111::2222:2222:2222", True), (r"1111:1111::2222:2222:2222:2222", True), (r"1111::2222:2222:2222:2222:2222", True), (r"1111:1111:1111:1111:1111:1111::2222", True), (r"1111:1111:1111:1111:1111::2222:2222", True), (r"1111:1111:1111:1111::2222:2222:2222", True), (r"1111:1111:1111::2222:2222:2222:2222", True), (r"1111:1111::2222:2222:2222:2222:2222", True), (r"1111::2222:2222:2222:2222:2222:2222", True), (r"2001:db8:3:4::192.0.2.33", True), (r"64:ff9b::192.0.2.33", True), (r"::ffff:0:255.255.255.255", True), (r"::111.111.222.222", True), (r":", False), (r":::", False), (r"::5555:5555:5555:5555:5555:5555:5555:5555", False), (r"5555::5555:5555:5555:5555:5555:5555:5555", False), (r"5555:5555::5555:5555:5555:5555:5555:5555", False), (r"5555:5555:5555::5555:5555:5555:5555:5555", False), (r"5555:5555:5555:5555::5555:5555:5555:5555", False), (r"5555:5555:5555:5555:5555::5555:5555:5555", False), (r"5555:5555:5555:5555:5555:5555::5555:5555", False), (r"5555:5555:5555:5555:5555:5555:5555::5555", False), (r"5555:5555:5555:5555:5555:5555:5555:5555::", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_ipv6_format) def test_ipv6_format(instance: str, accepted: bool): schema = {"type": "string", "format": "ipv6"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" ( ( [0-9a-fA-F]{1,4} ":" ){7,7} [0-9a-fA-F]{1,4} | ( [0-9a-fA-F]{1,4} ":" ){1,7} ":" | ( [0-9a-fA-F]{1,4} ":" ){1,6} ":" [0-9a-fA-F]{1,4} | ( [0-9a-fA-F]{1,4} ":" ){1,5} ( ":" [0-9a-fA-F]{1,4} ){1,2} | ( [0-9a-fA-F]{1,4} ":" ){1,4} ( ":" [0-9a-fA-F]{1,4} ){1,3} | ( [0-9a-fA-F]{1,4} ":" ){1,3} ( ":" [0-9a-fA-F]{1,4} ){1,4} | ( [0-9a-fA-F]{1,4} ":" ){1,2} ( ":" [0-9a-fA-F]{1,4} ){1,5} | [0-9a-fA-F]{1,4} ":" ( ( ":" [0-9a-fA-F]{1,4} ){1,6} ) | ":" ( ( ":" [0-9a-fA-F]{1,4} ){1,7} | ":" ) | ":" ":" ( "f" "f" "f" "f" ( ":" "0"{1,4} ){0,1} ":" ){0,1} ( ( "2" "5" [0-5] | ( "2" [0-4] | "1"{0,1} [0-9] ){0,1} [0-9] ) "." ){3,3} ( "2" "5" [0-5] | ( "2" [0-4] | "1"{0,1} [0-9] ){0,1} [0-9] ) | ( [0-9a-fA-F]{1,4} ":" ){1,4} ":" ( ( "2" "5" [0-5] | ( "2" [0-4] | "1"{0,1} [0-9] ){0,1} [0-9] ) "." ){3,3} ( "2" "5" [0-5] | ( "2" [0-4] | "1"{0,1} [0-9] ){0,1} [0-9] ) ) "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_ipv4_format = [ # (r"0.0.0.0", True), (r"00.00.00.00", True), (r"000.000.000.000", True), (r"255.255.255.255", True), (r"1", False), (r"1.", False), (r"1.1", False), (r"1.1.", False), (r"1.1.1", False), (r"1.1.1.", False), (r"0001.0001.0001.0001", False), (r"256.256.256.256", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_ipv4_format) def test_ipv4_format(instance: str, accepted: bool): schema = {"type": "string", "format": "ipv4"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" ( ( "2" "5" [0-5] | "2" [0-4] [0-9] | [0-1]? [0-9]? [0-9] ) "." ){3} ( "2" "5" [0-5] | "2" [0-4] [0-9] | [0-1]? [0-9]? [0-9] ) "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_hostname_format = [ (r"0", True), (r"9", True), (r"a", True), (r"z", True), (r"www.github.com", True), (r"w-w-w.g-i-t-h-u-b.c-o-m", True), (r"ww-w.gi-th-ub.co-m", True), (r"w--ww.git---hub.co----m", True), (r".", False), (r"-", False), (r"-.", False), (r".-", False), (r"_", False), (r"a.", False), (r"-b", False), (r"c-", False), (r"d.-", False), (r"e-.", False), (r"-f.", False), (r"g-.h", False), (r"-i.j", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_hostname_format) def test_hostname_format(instance: str, accepted: bool): schema = {"type": "string", "format": "hostname"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" ( [a-z0-9] ( [a-z0-9-]* [a-z0-9] )? ) ( "." [a-z0-9] ( [a-z0-9-]* [a-z0-9] )? )* "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_uuid_format = [ (r"00000000-0000-0000-0000-000000000000", True), (r"FFFFFFFF-FFFF-FFFF-FFFF-FFFFFFFFFFFF", True), (r"01234567-89AB-CDEF-abcd-ef0123456789", True), (r"-", False), (r"----", False), (r"AAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA", False), (r"BBBBBBBB-BBB-BBBB-BBBB-BBBBBBBBBBBB", False), (r"CCCCCCCC-CCCC-CCC-CCCC-CCCCCCCCCCCC", False), (r"DDDDDDDD-DDDD-DDDD-DDD-DDDDDDDDDDDD", False), (r"EEEEEEEE-EEEE-EEEE-EEEE-EEEEEEEEEEE", False), (r"AAAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA", False), (r"BBBBBBBB-BBBBB-BBBB-BBBB-BBBBBBBBBBBB", False), (r"CCCCCCCC-CCCC-CCCCC-CCCC-CCCCCCCCCCCC", False), (r"DDDDDDDD-DDDD-DDDD-DDDDD-DDDDDDDDDDDD", False), (r"EEEEEEEE-EEEE-EEEE-EEEE-EEEEEEEEEEEEE", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_uuid_format) def test_uuid_format(instance: str, accepted: bool): schema = {"type": "string", "format": "uuid"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" [0-9A-Fa-f]{8} "-" [0-9A-Fa-f]{4} "-" [0-9A-Fa-f]{4} "-" [0-9A-Fa-f]{4} "-" [0-9A-Fa-f]{12} "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_uri_format = [ (r"aaa:?azAZ09-._~%Ff!$&'()*+,;=:@#azAZ09-._~%Aa!$&'()*+,;=:@", True), (r"z+.-:", True), (r"abc:", True), (r"abc:a", True), (r"abc:/", True), (r"abc:/a", True), (r"abc://", True), (r"abc://///////", True), (r"abc://azAZ09-._~%Ff!$&'()*+,;=:@", True), (r"abc://:", True), (r"abc://:0123", True), (r"abc://azAZ09-._~%Ff!$&'()*+,;=", True), (r"xyz:/a", True), (r"xyz:/azAZ09-._~%Ff!$&'()*+,;=:@", True), (r"aaa:?[#]", False), (r"abc://@@", False), (r"abc://::", False), (r"abc:/[]", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_uri_format) def test_uri_format(instance: str, accepted: bool): schema = {"type": "string", "format": "uri"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" [a-zA-Z] [a-zA-Z+.-]* ":" ( "/" "/" ( ( [a-zA-Z0-9_.~!$&'()*+,;=:-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )* "@" )? ( [a-zA-Z0-9_.~!$&'()*+,;=-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )* ( ":" [0-9]* )? ( "/" ( [a-zA-Z0-9_.~!$&'()*+,;=:@-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )* )* | "/"? ( ( [a-zA-Z0-9_.~!$&'()*+,;=:@-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )+ ( "/" ( [a-zA-Z0-9_.~!$&'()*+,;=:@-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )* )* )? ) ( "\?" ( [a-zA-Z0-9_.~!$&'()*+,;=:@/\?-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )* )? ( "#" ( [a-zA-Z0-9_.~!$&'()*+,;=:@/\?-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )* )? "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_uri_reference_format = [ (r"?azAZ09-._~%Ff!$&'()*+,;=:@#azAZ09-._~%Aa!$&'()*+,;=:@", True), (r"", True), (r"a", True), (r"/", True), (r"/a", True), (r"//", True), (r"/////////", True), (r"//azAZ09-._~%Ff!$&'()*+,;=:@", True), (r"//:", True), (r"//:0123", True), (r"//azAZ09-._~%Ff!$&'()*+,;=", True), (r"/a", True), (r"/azAZ09-._~%Ff!$&'()*+,;=:@", True), (r"?[#]", False), (r"//@@", False), (r"//::", False), (r"/[]", False), (r":", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_uri_reference_format) def test_uri_reference_format(instance: str, accepted: bool): schema = {"type": "string", "format": "uri-reference"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" ( "/" "/" ( ( [a-zA-Z0-9_.~!$&'()*+,;=:-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )* "@" )? ( [a-zA-Z0-9_.~!$&'()*+,;=-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )* ( ":" [0-9]* )? ( "/" ( [a-zA-Z0-9_.~!$&'()*+,;=:@-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )* )* | "/" ( ( [a-zA-Z0-9_.~!$&'()*+,;=:@-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )+ ( "/" ( [a-zA-Z0-9_.~!$&'()*+,;=:@-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )* )* )? | ( [a-zA-Z0-9_.~!$&'()*+,;=@-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )+ ( "/" ( [a-zA-Z0-9_.~!$&'()*+,;=:@-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )* )* )? ( "\?" ( [a-zA-Z0-9_.~!$&'()*+,;=:@/\?-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )* )? ( "#" ( [a-zA-Z0-9_.~!$&'()*+,;=:@/\?-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )* )? "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_uri_template_format = [ (r"", True), (r"!#$&()*+,-./09:;=?@AZ[]_az~%Ff", True), (r"{+a}{#a}{.a}{/a}{;a}{?a}{&a}{=a}{,a}{!a}{@a}{|a}", True), (r"{%Ff}", True), (r"{i.j.k}", True), (r"{a_b_c:1234}", True), (r"{x_y_z*}", True), (r'"', False), (r"'", False), (r"%", False), (r"<", False), (r">", False), (r"\\\\", False), (r"^", False), (r"`", False), (r"{", False), (r"|", False), (r"}", False), (r"{n.}", False), (r"{m:100001}", False), (r"%1", False), (r"%Gg", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_uri_template_format) def test_uri_template_format(instance: str, accepted: bool): schema = {"type": "string", "format": "uri-template"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" ( ( [!#-$&(-;=\?-[\]_a-z~] | "%" [0-9A-Fa-f] [0-9A-Fa-f] ) | "{" ( [+#./;\?&=,!@|] )? ( [a-zA-Z0-9_] | "%" [0-9A-Fa-f] [0-9A-Fa-f] ) ( "."? ( [a-zA-Z0-9_] | "%" [0-9A-Fa-f] [0-9A-Fa-f] ) )* ( ":" [1-9] [0-9]? [0-9]? [0-9]? | "*" )? ( "," ( [a-zA-Z0-9_] | "%" [0-9A-Fa-f] [0-9A-Fa-f] ) ( "."? ( [a-zA-Z0-9_] | "%" [0-9A-Fa-f] [0-9A-Fa-f] ) )* ( ":" [1-9] [0-9]? [0-9]? [0-9]? | "*" )? )* "}" )* "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_json_pointer_format = [ (r"/", True), (r"//", True), (r"/a/bc/def/ghij", True), (r"/~0/~1/", True), (r"abc", False), (r"/~", False), (r"/~2", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_json_pointer_format) def test_json_pointer_format(instance: str, accepted: bool): schema = {"type": "string", "format": "json-pointer"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" ( "/" ( [\0-.] | [0-}] | [\x7f-\U0010ffff] | "~" [01] )* )* "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_relative_json_pointer_format = [ (r"0/", True), (r"123/a/bc/def/ghij", True), (r"45/~0/~1/", True), (r"6789#", True), (r"#", False), (r"abc", False), (r"/", False), (r"9/~2", False), ] @pytest.mark.parametrize( "instance, accepted", instance__accepted__test_relative_json_pointer_format ) def test_relative_json_pointer_format(instance: str, accepted: bool): schema = {"type": "string", "format": "relative-json-pointer"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" ( "0" | [1-9] [0-9]* ) ( "#" | ( "/" ( [\0-.] | [0-}] | [\x7f-\U0010ffff] | "~" [01] )* )* ) "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) def test_min_max_length(): schema = {"type": "string", "minLength": 1, "maxLength": 10} ebnf_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" [^"\\\r\n]{1,10} "\"" """ ) check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=True) instance_accepted = '"abcdefghij"' instance_rejected = '"abcdefghijk"' check_schema_with_instance(schema, instance_accepted, any_whitespace=True) check_schema_with_instance(schema, instance_rejected, is_accepted=False, any_whitespace=True) def test_type_array(): schema = { "type": ["integer", "string"], "minLength": 1, "maxLength": 10, "minimum": 1, "maximum": 10, } ebnf_grammar = basic_json_rules_ebnf + ( r"""root_type_0 ::= ( ( [1-9] | "1" "0" ) ) root_type_1 ::= "\"" [^"\\\r\n]{1,10} "\"" root ::= root_type_0 | root_type_1 """ ) check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=True) instance_accepted = "1" instance_accepted_2 = '"1234567890"' instance_rejected = "11" instance_rejected_2 = '"12345678901"' check_schema_with_instance(schema, instance_accepted, any_whitespace=True) check_schema_with_instance(schema, instance_accepted_2, any_whitespace=True) check_schema_with_instance(schema, instance_rejected, is_accepted=False, any_whitespace=True) check_schema_with_instance(schema, instance_rejected_2, is_accepted=False, any_whitespace=True) def test_type_array_empty(): schema = {"type": []} ebnf_grammar = basic_json_rules_ebnf + ( r"""root ::= basic_any """ ) check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=True) def test_empty_array(): schema = {"items": {"type": "string"}, "type": "array"} ebnf_grammar = basic_json_rules_ebnf + ( r"""root ::= (("[" [ \n\t]* basic_string ([ \n\t]* "," [ \n\t]* basic_string)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) """ ) check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=True) instance_accepted = "[]" instance_accepted_2 = '["a"]' check_schema_with_instance(schema, instance_accepted, any_whitespace=True) check_schema_with_instance(schema, instance_accepted_2, any_whitespace=True) def test_empty_object(): schema = {"properties": {"name": {"type": "string"}}, "type": "object"} ebnf_grammar = basic_json_rules_ebnf + ( r"""root ::= ("{" [ \n\t]* (("\"name\"" [ \n\t]* ":" [ \n\t]* basic_string "")) [ \n\t]* "}") | "{" [ \n\t]* "}" """ ) check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=True) instance_accepted = "{}" instance_accepted_2 = '{"name": "test"}' check_schema_with_instance(schema, instance_accepted, any_whitespace=True) check_schema_with_instance(schema, instance_accepted_2, any_whitespace=True) def test_primitive_type_string(): schema = {"type": "string"} ebnf_grammar = basic_json_rules_ebnf + ( r"""root ::= basic_string """ ) check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=True) instance_accepted = '"test"' instance_rejected = "123" check_schema_with_instance(schema, instance_accepted, any_whitespace=True) check_schema_with_instance(schema, instance_rejected, is_accepted=False, any_whitespace=True) def test_primitive_type_object(): schema = {"type": "object"} ebnf_grammar = basic_json_rules_ebnf + ( r"""root ::= basic_object """ ) check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=True) instance_accepted = '{"name": "test"}' instance_rejected = '"test"' check_schema_with_instance(schema, instance_accepted, any_whitespace=True) check_schema_with_instance(schema, instance_rejected, is_accepted=False, any_whitespace=True) def test_generate_float_regex(): assert ( _generate_float_regex(1.0, 5.0) == r"^(1\.[1-9]\d{0,5}|1\.0[1-9]\d{0,4}|1\.00[1-9]\d{0,3}|1\.000[1-9]\d{0,2}|1\.0000[1-9]\d{0,1}|1\.00000[1-9]|1\.0{1,6}|1|(([2-4]))(\.\d{1,6})?|5\.0{1,6}|5)$" ) assert ( _generate_float_regex(1.5, 5.75) == r"^(1\.[6-9]\d{0,5}|1\.5[1-9]\d{0,4}|1\.50[1-9]\d{0,3}|1\.500[1-9]\d{0,2}|1\.5000[1-9]\d{0,1}|1\.50000[1-9]|1\.50{0,5}|(([2-4]))(\.\d{1,6})?|5\.[0-6]\d{0,5}|5\.7[0-4]\d{0,4}|5\.0{1,6}|5\.70{0,5}|5\.750{0,4}|5)$" ) assert ( _generate_float_regex(-3.14, 2.71828) == r"^(-0\.[1-9]\d{0,5}|-0\.0[1-9]\d{0,4}|-0\.00[1-9]\d{0,3}|-0\.000[1-9]\d{0,2}|-0\.0000[1-9]\d{0,1}|-0\.00000[1-9]|-(([1-2]))(\.\d{1,6})?|-3\.0\d{0,5}|-3\.1[0-3]\d{0,4}|-3\.0{1,6}|-3\.10{0,5}|-3\.140{0,4}|-3|0(\.0{1,6})?|-0(\.0{1,6})|0\.[1-9]\d{0,5}|0\.0[1-9]\d{0,4}|0\.00[1-9]\d{0,3}|0\.000[1-9]\d{0,2}|0\.0000[1-9]\d{0,1}|0\.00000[1-9]|((1))(\.\d{1,6})?|2\.[0-6]\d{0,5}|2\.70\d{0,4}|2\.71[0-7]\d{0,3}|2\.718[0-1]\d{0,2}|2\.7182[0-7]\d{0,1}|2\.0{1,6}|2\.70{0,5}|2\.710{0,4}|2\.7180{0,3}|2\.71820{0,2}|2\.718280{0,1}|2)$" ) assert ( _generate_float_regex(0.5, None) == r"^(0\.[6-9]\d{0,5}|0\.5[1-9]\d{0,4}|0\.50[1-9]\d{0,3}|0\.500[1-9]\d{0,2}|0\.5000[1-9]\d{0,1}|0\.50000[1-9]|0\.50{0,5}|([1-9]|[1-9]\d{1,})(\.\d{1,6})?)$" ) assert ( _generate_float_regex(None, -1.5) == r"^(-1\.[6-9]\d{0,5}|-1\.5[1-9]\d{0,4}|-1\.50[1-9]\d{0,3}|-1\.500[1-9]\d{0,2}|-1\.5000[1-9]\d{0,1}|-1\.50000[1-9]|-1\.50{0,5}|-([2-9]|[1-9]\d{1,})(\.\d{1,6})?)$" ) assert _generate_float_regex(None, None) == r"^-?\d+(\.\d{1,6})?$" assert _generate_float_regex(3.14159, 3.14159) == r"^(3\.141590{0,1})$" assert _generate_float_regex(10.5, 2.5) == r"^()$" assert _generate_float_regex(5.123456, 5.123457) == r"^(5\.123456|5\.123457)$" assert ( _generate_float_regex(-0.000001, 0.000001) == r"^(-0\.000001|0(\.0{1,6})?|-0(\.0{1,6})|0\.000001)$" ) # exclusive bounds drop the boundary value itself assert ( _generate_float_regex(0, None, exclusive_start=True) == r"^(0\.[1-9]\d{0,5}|0\.0[1-9]\d{0,4}|0\.00[1-9]\d{0,3}|0\.000[1-9]\d{0,2}|0\.0000[1-9]\d{0,1}|0\.00000[1-9]|([1-9]|[1-9]\d{1,})(\.\d{1,6})?)$" ) assert _generate_float_regex(0, None) == ( r"^(0(\.0{1,6})?|0\.[1-9]\d{0,5}|0\.0[1-9]\d{0,4}|0\.00[1-9]\d{0,3}|0\.000[1-9]\d{0,2}|0\.0000[1-9]\d{0,1}|0\.00000[1-9]|([1-9]|[1-9]\d{1,})(\.\d{1,6})?)$" ) assert _generate_float_regex(2.5, 2.5, exclusive_end=True) == r"^()$" def test_generate_float_regex_cross_zero_accepts_negative_zero_decimal(): regex = re.compile(_generate_float_regex(-4.0, 4.0)) for value in ("-0.1", "-0.5", "-0.999999"): assert regex.fullmatch(value) is not None # negative zero written with an all-zero fraction denotes 0, which is in range for value in ("-0.0", "-0.000000"): assert regex.fullmatch(value) is not None assert regex.fullmatch("-0") is None assert regex.fullmatch("-4.1") is None assert regex.fullmatch("4.1") is None near_zero_regex = re.compile(_generate_float_regex(-0.5, 0.5)) assert near_zero_regex.fullmatch("-0.1") is not None assert near_zero_regex.fullmatch("-0.5") is not None assert near_zero_regex.fullmatch("-0.9") is None schema = {"type": "number", "minimum": -4.0, "maximum": 4.0} check_schema_with_instance(schema, "-0.5") check_schema_with_instance(schema, "-0.1") check_schema_with_instance(schema, "-4.1", is_accepted=False) check_schema_with_instance(schema, "4.1", is_accepted=False) near_zero_schema = {"type": "number", "minimum": -0.5, "maximum": 0.5} check_schema_with_instance(near_zero_schema, "-0.1") check_schema_with_instance(near_zero_schema, "-0.5") check_schema_with_instance(near_zero_schema, "-0.9", is_accepted=False) def test_generate_float_regex_one_sided_integer_boundaries(): minimum_regex = re.compile(_generate_float_regex(4.0, None)) assert minimum_regex.fullmatch("4.1") is not None assert minimum_regex.fullmatch("4.999999") is not None assert minimum_regex.fullmatch("3.999999") is None maximum_regex = re.compile(_generate_float_regex(None, -4.0)) assert maximum_regex.fullmatch("-4.1") is not None assert maximum_regex.fullmatch("-4.999999") is not None assert maximum_regex.fullmatch("-3.999999") is None check_schema_with_instance({"type": "number", "minimum": 4.0}, "4.1") check_schema_with_instance({"type": "number", "minimum": 4.0}, "3.9", is_accepted=False) check_schema_with_instance({"type": "number", "maximum": -4.0}, "-4.1") check_schema_with_instance({"type": "number", "maximum": -4.0}, "-3.9", is_accepted=False) def test_generate_float_regex_fractional_upper_bound_includes_floor_integer(): positive_regex = re.compile(_generate_float_regex(1.5, 5.75)) assert positive_regex.fullmatch("5") is not None assert positive_regex.fullmatch("5.75") is not None assert positive_regex.fullmatch("6") is None negative_regex = re.compile(_generate_float_regex(None, -1.5)) assert negative_regex.fullmatch("-2") is not None assert negative_regex.fullmatch("-2.0") is not None assert negative_regex.fullmatch("-1") is None mixed_regex = re.compile(_generate_float_regex(-3.14, 2.71828)) assert mixed_regex.fullmatch("2") is not None assert mixed_regex.fullmatch("2.71828") is not None assert mixed_regex.fullmatch("3") is None def test_float_minimum_no_wildcard_in_grammar(): """Float minimum/maximum boundary values should not produce regex wildcard in grammar.""" schema = '{"type":"number","minimum":0.5}' grammar = xgr.Grammar.from_json_schema(schema) grammar_str = str(grammar) # The root rule should use literal "." not wildcard [\0-\U0010ffff] for line in grammar_str.split("\n"): if line.startswith("root"): assert "[\\0-\\U0010ffff]" not in line, f"Wildcard found in: {line}" schema2 = '{"type":"number","maximum":9.5}' grammar2 = xgr.Grammar.from_json_schema(schema2) for line in str(grammar2).split("\n"): if line.startswith("root"): assert "[\\0-\\U0010ffff]" not in line, f"Wildcard found in: {line}" schema3 = '{"type":"number","minimum":0.5,"maximum":9.5}' grammar3 = xgr.Grammar.from_json_schema(schema3) for line in str(grammar3).split("\n"): if line.startswith("root"): assert "[\\0-\\U0010ffff]" not in line, f"Wildcard found in: {line}" number_range_instances = [ # exclusiveMinimum with an integer-valued bound: (0, 1) must be representable, bound rejected ({"type": "number", "exclusiveMinimum": 0}, "0.1", True), ({"type": "number", "exclusiveMinimum": 0}, "0", False), ({"type": "number", "minimum": 0}, "0", True), ({"type": "number", "minimum": 0}, "-0.5", False), # minimum above 1 ({"type": "number", "minimum": 2}, "1.5", False), ({"type": "number", "minimum": 2}, "2", True), # upper bounds ({"type": "number", "exclusiveMaximum": 1}, "1", False), ({"type": "number", "exclusiveMaximum": 1}, "0.99", True), ({"type": "number", "maximum": -2}, "-1.5", False), ({"type": "number", "maximum": -2}, "-2", True), # both bounds: a value above the maximum must be rejected ({"type": "number", "minimum": 1, "maximum": 5}, "5.7", False), ({"type": "number", "minimum": 1, "maximum": 5}, "5", True), ({"type": "number", "exclusiveMinimum": 1, "exclusiveMaximum": 5}, "1", False), ({"type": "number", "minimum": 0.1, "maximum": 0.3}, "0.2", True), ({"type": "number", "minimum": 0.1, "maximum": 0.3}, "0.35", False), # multi-digit integer part must not leak (regression: 159.5 over-accepted) ({"type": "number", "minimum": 140, "maximum": 159}, "159", True), ({"type": "number", "minimum": 140, "maximum": 159}, "159.5", False), ({"type": "number", "minimum": 140, "maximum": 159}, "149.5", True), # fractional-bound boundaries earlier patch-style generators got wrong ({"type": "number", "minimum": -3.14, "maximum": 2.71828}, "-3.9", False), ({"type": "number", "minimum": 0.1, "maximum": 0.5}, "0.2", True), ({"type": "number", "minimum": -0.5, "maximum": 0.5}, "-0.9", False), # an integer-valued bound must admit/reject fractions on the correct side ({"type": "number", "minimum": 4.0}, "4.1", True), ({"type": "number", "minimum": 4.0}, "3.9", False), ({"type": "number", "maximum": -4.0}, "-4.1", True), # both minimum and exclusiveMinimum: the stricter bound wins ({"type": "number", "minimum": 5, "exclusiveMinimum": 3}, "4", False), ({"type": "number", "minimum": 3, "exclusiveMinimum": 3}, "3", False), ({"type": "number", "maximum": 3, "exclusiveMaximum": 3}, "3", False), # mixed inclusive/exclusive ({"type": "number", "minimum": 2, "exclusiveMaximum": 5}, "5", False), ({"type": "number", "exclusiveMinimum": 2, "maximum": 5}, "2", False), # single-value range ({"type": "number", "minimum": 5, "maximum": 5}, "5", True), ({"type": "number", "minimum": 5, "maximum": 5}, "5.000001", False), # negative exclusive ({"type": "number", "exclusiveMinimum": -5.5}, "-5.5", False), ({"type": "number", "exclusiveMinimum": -5.5}, "-5.499999", True), # bounds with more fraction digits than the 6-digit precision must round # toward the feasible region (upper rounds down, lower rounds up) so no # out-of-range value leaks in ({"type": "number", "maximum": 0.9999999}, "1", False), ({"type": "number", "maximum": 0.9999999}, "0.999999", True), ({"type": "number", "maximum": 4.9999996}, "5", False), ({"type": "number", "maximum": 0.0000006}, "0.000001", False), ({"type": "number", "maximum": 0.0000006}, "0", True), ({"type": "number", "minimum": 1.0000004}, "1", False), ({"type": "number", "minimum": 1.0000004}, "1.000001", True), ({"type": "number", "minimum": 5, "exclusiveMaximum": 5.0000001}, "5", True), # both bounds collapse onto the same grid point but the value is in range ({"type": "number", "minimum": 1, "exclusiveMaximum": 1.0000004}, "1", True), ({"type": "number", "exclusiveMinimum": 0.9999999, "maximum": 1}, "1", True), # large-magnitude bounds (>= 1e18) must not be clamped to ~1e18 ({"type": "number", "minimum": 5e18}, "1000000000000000000", False), ({"type": "number", "minimum": 5e18}, "6000000000000000000", True), ({"type": "number", "maximum": 1e19}, "5000000000000000000", True), ] @pytest.mark.parametrize("schema, instance, accepted", number_range_instances) def test_number_range_value_acceptance(schema, instance, accepted): check_schema_with_instance(schema, instance, is_accepted=accepted) unsatisfiable_range_schemas = [ # minimum greater than maximum {"type": "number", "minimum": 10, "maximum": 5}, {"type": "integer", "minimum": 10, "maximum": 5}, # min == max but the single candidate value is excluded by an exclusive bound {"type": "number", "exclusiveMinimum": 5, "exclusiveMaximum": 5}, {"type": "number", "minimum": 5, "exclusiveMaximum": 5}, {"type": "number", "exclusiveMinimum": 5, "maximum": 5}, {"type": "number", "minimum": 5.5, "exclusiveMaximum": 5.5}, {"type": "number", "minimum": 5, "exclusiveMinimum": 5, "maximum": 5}, {"type": "integer", "exclusiveMinimum": 5, "exclusiveMaximum": 6}, {"type": "integer", "minimum": 5, "exclusiveMaximum": 5}, ] @pytest.mark.parametrize("schema", unsatisfiable_range_schemas) def test_unsatisfiable_range_raises(schema): """An impossible numeric range must be rejected at build time.""" with pytest.raises(RuntimeError): xgr.Grammar.from_json_schema(json.dumps(schema)) integer_range_instances = [ # minimum above 1: single-digit integers below the bound must be rejected ({"type": "integer", "minimum": 2}, "1", False), ({"type": "integer", "minimum": 2}, "2", True), ({"type": "integer", "exclusiveMinimum": 2}, "2", False), ({"type": "integer", "exclusiveMinimum": 2}, "3", True), # negative maximum / negative minimum ({"type": "integer", "maximum": -2}, "-1", False), ({"type": "integer", "maximum": -2}, "-2", True), ({"type": "integer", "minimum": -5}, "-6", False), ({"type": "integer", "minimum": -5}, "-5", True), # multi-digit two-sided: a value above max sharing the lower bound's digits must be rejected ({"type": "integer", "minimum": 100, "maximum": 110}, "110", True), ({"type": "integer", "minimum": 100, "maximum": 110}, "111", False), # negative multi-digit maximum (regression: -11..-99 were dropped / over-accepted) ({"type": "integer", "maximum": -10}, "-11", True), ({"type": "integer", "maximum": -50}, "-49", False), ({"type": "integer", "maximum": -50}, "-51", True), # multi-digit positive minimum ({"type": "integer", "minimum": 100}, "99", False), ({"type": "integer", "minimum": 100}, "100", True), # int64 boundaries: negating INT64_MIN must not overflow ( {"type": "integer", "minimum": -9223372036854775808, "maximum": 0}, "-9223372036854775808", True, ), ( {"type": "integer", "minimum": 0, "maximum": 9223372036854775807}, "9223372036854775807", True, ), ( {"type": "integer", "minimum": 0, "maximum": 9223372036854775807}, "9223372036854775808", False, ), # both minimum and exclusiveMinimum: the stricter bound wins (regression: inclusive min discarded) ({"type": "integer", "minimum": 5, "exclusiveMinimum": 3}, "4", False), ({"type": "integer", "minimum": 5, "exclusiveMinimum": 3}, "5", True), ({"type": "integer", "maximum": 3, "exclusiveMaximum": 5}, "4", False), # single-value range ({"type": "integer", "minimum": 5, "maximum": 5}, "5", True), ({"type": "integer", "minimum": 5, "maximum": 5}, "4", False), # exclusive at a multi-digit boundary ({"type": "integer", "exclusiveMinimum": 99}, "99", False), ({"type": "integer", "exclusiveMinimum": 99}, "100", True), ] @pytest.mark.parametrize("schema, instance, accepted", integer_range_instances) def test_integer_range_value_acceptance(schema, instance, accepted): check_schema_with_instance(schema, instance, is_accepted=accepted) number_range_sweep_bounds = [ {"minimum": 0}, {"exclusiveMinimum": 0}, {"maximum": 0}, {"exclusiveMaximum": 0}, {"minimum": 2}, {"exclusiveMinimum": 2}, {"minimum": -2}, {"maximum": 5}, {"minimum": 0.5}, {"exclusiveMinimum": 0.5}, {"maximum": 0.5}, {"exclusiveMaximum": 0.5}, {"minimum": 99.5}, {"maximum": -2.25}, {"minimum": 1, "maximum": 5}, {"exclusiveMinimum": 1, "exclusiveMaximum": 5}, {"minimum": -1.5, "maximum": 1.5}, {"minimum": 0.1, "maximum": 0.3}, {"minimum": -5.5, "maximum": -2.25}, {"minimum": 9, "maximum": 31}, {"minimum": 5.123456, "maximum": 5.123457}, # multi-digit integer parts: stress the integer "middle" reuse {"minimum": 140, "maximum": 159}, {"minimum": 100, "maximum": 110}, {"minimum": -159, "maximum": -140}, {"minimum": -110, "maximum": -100}, {"minimum": 99, "maximum": 101}, {"minimum": 12.5, "maximum": 130.25}, {"exclusiveMinimum": 100, "exclusiveMaximum": 110}, {"maximum": -10.5}, {"minimum": 1000.5}, {"minimum": -120, "maximum": 120}, # fractional bounds with non-trivial boundary fractions on both sides {"minimum": -3.14, "maximum": 2.71828}, {"minimum": 0.1, "maximum": 0.5}, {"minimum": -0.5, "maximum": 0.5}, {"minimum": 4.0}, {"maximum": -4.0}, {"minimum": -4, "maximum": 4}, {"minimum": 5, "exclusiveMinimum": 3}, {"maximum": 3, "exclusiveMaximum": 5}, {"minimum": 1, "exclusiveMinimum": 2, "maximum": 9, "exclusiveMaximum": 8}, ] @pytest.mark.parametrize("bounds", number_range_sweep_bounds) def test_number_range_acceptance_sweep(bounds): """The grammar for a range-constrained number must agree with plain float comparison for every candidate value around the bounds (limited to 6 fractional digits, the converter's precision).""" def in_range(value: float) -> bool: if "minimum" in bounds and not value >= bounds["minimum"]: return False if "exclusiveMinimum" in bounds and not value > bounds["exclusiveMinimum"]: return False if "maximum" in bounds and not value <= bounds["maximum"]: return False if "exclusiveMaximum" in bounds and not value < bounds["exclusiveMaximum"]: return False return True candidates = {0.0, 1.0, -1.0, 0.5, -0.5, 10.0, -10.0, 100.0, -100.0} for bound in bounds.values(): # Larger deltas reach the multi-digit interior on the unbounded side of # one-sided ranges (where the dense floor-loop below cannot help). for delta in (0.0, 0.000001, 0.1, 0.5, 1.0, 2.0, 10.0, 37.0, 123.5, 1234.0): candidates.add(bound + delta) candidates.add(bound - delta) # Densely cover the interior of bounded ranges so multi-digit integer parts # (the integer "middle" of the float range) are exercised, not just the # immediate neighbourhood of each bound. numeric = list(bounds.values()) lo_i = int(min(numeric)) - 3 hi_i = int(max(numeric)) + 3 if hi_i - lo_i <= 400: for k in range(lo_i, hi_i + 1): candidates.add(float(k)) candidates.add(k + 0.5) grammar = xgr.Grammar.from_json_schema(json.dumps({"type": "number", **bounds})) for value in sorted(candidates): text = f"{value:.6f}".rstrip("0").rstrip(".") if text in ("", "-0"): text = "0" value = float(text) accepted = _is_grammar_accept_string(grammar, text) assert accepted == in_range(value), ( f"bounds={bounds} value={text}: grammar " f"{'accepted' if accepted else 'rejected'}, float comparison says " f"{'in range' if in_range(value) else 'out of range'}" ) integer_range_sweep_bounds = [ {"minimum": 2}, {"exclusiveMinimum": 2}, {"maximum": -2}, {"minimum": -5}, {"minimum": 100}, {"maximum": -10}, {"maximum": -50}, {"maximum": -99}, {"maximum": -100}, {"minimum": 100, "maximum": 110}, {"minimum": 0, "maximum": 9}, {"minimum": 78, "maximum": 1278}, {"minimum": -120, "maximum": 120}, {"minimum": -1999, "maximum": -100}, {"minimum": 5, "maximum": 100}, {"minimum": 999, "maximum": 1001}, {"minimum": 95, "maximum": 105}, {"minimum": -10, "maximum": -5}, {"exclusiveMinimum": 9, "exclusiveMaximum": 31}, {"minimum": 12345, "maximum": 54321}, {"minimum": 10000000000}, {"minimum": 5, "exclusiveMinimum": 3}, {"maximum": 3, "exclusiveMaximum": 5}, {"minimum": 1, "exclusiveMinimum": 2, "maximum": 9, "exclusiveMaximum": 8}, ] @pytest.mark.parametrize("bounds", integer_range_sweep_bounds) def test_integer_range_acceptance_sweep(bounds): """The grammar for a range-constrained integer must agree with plain integer comparison for every candidate value around the bounds.""" def in_range(value: int) -> bool: if "minimum" in bounds and not value >= bounds["minimum"]: return False if "exclusiveMinimum" in bounds and not value > bounds["exclusiveMinimum"]: return False if "maximum" in bounds and not value <= bounds["maximum"]: return False if "exclusiveMaximum" in bounds and not value < bounds["exclusiveMaximum"]: return False return True candidates = set(range(-30, 31)) for bound in bounds.values(): for delta in range(-12, 13): candidates.add(bound + delta) candidates |= {bound * 10, bound * 100, -bound} candidates |= {0, 999, 1000, 1001, -999, -1000, -1001, 12344, 12345, 54321, 54322} grammar = xgr.Grammar.from_json_schema(json.dumps({"type": "integer", **bounds})) for value in sorted(candidates): text = str(value) accepted = _is_grammar_accept_string(grammar, text) assert accepted == in_range(value), ( f"bounds={bounds} value={text}: grammar " f"{'accepted' if accepted else 'rejected'}, integer comparison says " f"{'in range' if in_range(value) else 'out of range'}" ) def test_limited_whitespace_cnt(): expected_grammar = r"""basic_escape ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) (=(basic_string_sub)) basic_string_sub ::= (("\"") | ([^\0-\x1f\"\\\r\n] basic_string_sub) | ("\\" basic_escape basic_string_sub)) (=(basic_string_sub_4 [,}\]:])) basic_string ::= (("\"" basic_string_sub)) (=(root_16 "}")) root ::= (("{" root_7 "\"key\"" root_10 ":" root_13 basic_string root_16 "}")) basic_string_sub_2 ::= ("" | ([ \n\t] basic_string_sub_3)) basic_string_sub_3 ::= ("" | ([ \n\t])) basic_string_sub_4 ::= ((basic_string_sub_2)) root_5 ::= ("" | ([ \n\t] root_6)) root_6 ::= ("" | ([ \n\t])) root_7 ::= ((root_5)) (=("\"key\"" root_10 ":" root_13 basic_string root_16 "}")) root_8 ::= ("" | ([ \n\t] root_9)) root_9 ::= ("" | ([ \n\t])) root_10 ::= ((root_8)) (=(":" root_13 basic_string root_16 "}")) root_11 ::= ("" | ([ \n\t] root_12)) root_12 ::= ("" | ([ \n\t])) root_13 ::= ((root_11)) (=(basic_string root_16 "}")) root_14 ::= ("" | ([ \n\t] root_15)) root_15 ::= ("" | ([ \n\t])) root_16 ::= ((root_14)) (=("}")) """ schema = {"type": "object", "properties": {"key": {"type": "string"}}, "required": ["key"]} grammar = xgr.Grammar.from_json_schema(schema, any_whitespace=True, max_whitespace_cnt=2) grammar = GrammarFunctor.grammar_optimizer(grammar) assert grammar is not None assert str(grammar) == expected_grammar assert _is_grammar_accept_string(grammar, '{ "key" : "value" }') assert _is_grammar_accept_string(grammar, '{"key":"value"}') assert not _is_grammar_accept_string(grammar, '{ "key" : "value" }') assert not _is_grammar_accept_string(grammar, '{ "key" : "value" }') def test_limited_whitespace_compile(): expected_grammar = r"""basic_escape ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) (=(basic_string_sub)) basic_string_sub ::= (("\"") | ([^\0-\x1f\"\\\r\n] basic_string_sub) | ("\\" basic_escape basic_string_sub)) (=(basic_string_sub_4 [,}\]:])) basic_string ::= (("\"" basic_string_sub)) (=(root_16 "}")) root ::= (("{" root_7 "\"key\"" root_10 ":" root_13 basic_string root_16 "}")) basic_string_sub_2 ::= ("" | ([ \n\t] basic_string_sub_3)) basic_string_sub_3 ::= ("" | ([ \n\t])) basic_string_sub_4 ::= ((basic_string_sub_2)) root_5 ::= ("" | ([ \n\t] root_6)) root_6 ::= ("" | ([ \n\t])) root_7 ::= ((root_5)) (=("\"key\"" root_10 ":" root_13 basic_string root_16 "}")) root_8 ::= ("" | ([ \n\t] root_9)) root_9 ::= ("" | ([ \n\t])) root_10 ::= ((root_8)) (=(":" root_13 basic_string root_16 "}")) root_11 ::= ("" | ([ \n\t] root_12)) root_12 ::= ("" | ([ \n\t])) root_13 ::= ((root_11)) (=(basic_string root_16 "}")) root_14 ::= ("" | ([ \n\t] root_15)) root_15 ::= ("" | ([ \n\t])) root_16 ::= ((root_14)) (=("}")) """ schema = {"type": "object", "properties": {"key": {"type": "string"}}, "required": ["key"]} tokenizer_info = xgr.TokenizerInfo([]) compiler = xgr.GrammarCompiler(tokenizer_info) compiled_grammar = compiler.compile_json_schema( schema, any_whitespace=True, max_whitespace_cnt=2 ) assert compiled_grammar is not None grammar = compiled_grammar.grammar assert str(grammar) == expected_grammar, str(grammar) assert grammar is not None assert _is_grammar_accept_string(grammar, '{ "key" : "value" }') assert _is_grammar_accept_string(grammar, '{"key":"value"}') assert not _is_grammar_accept_string(grammar, '{ "key" : "value" }') assert not _is_grammar_accept_string(grammar, '{ "key" : "value" }') def test_utf8_in_enum(): schema = {"type": "string", "enum": ["こんにちは", "😊", "你好", "hello", "\n"]} grammar = xgr.Grammar.from_json_schema(schema) assert _is_grammar_accept_string(grammar, '"こんにちは"') assert _is_grammar_accept_string(grammar, '"😊"') assert _is_grammar_accept_string(grammar, '"你好"') assert _is_grammar_accept_string(grammar, '"hello"') assert _is_grammar_accept_string(grammar, '"\\n"') def test_utf8_string_in_const(): schema = {"const": "常数constじょうすう\n\r\t"} grammar = xgr.Grammar.from_json_schema(schema) assert _is_grammar_accept_string(grammar, '"常数constじょうすう\\n\\r\\t"') def test_control_char_in_property_key(): vocab = [bytes([i]) for i in range(256)] metadata = json.dumps( { "vocab_type": 0, "vocab_size": 256, "prepend_space_in_tokenization": False, "add_prefix_space": False, "stop_token_ids": [0], } ) tokenizer_info = xgr.TokenizerInfo.from_vocab_and_metadata(vocab, metadata) schema = { "type": "object", "properties": {"key\x01ctrl": {"type": "string"}}, "required": ["key\x01ctrl"], } compiler = xgr.GrammarCompiler(tokenizer_info) grammar = compiler.compile_json_schema(json.dumps(schema)) matcher = xgr.GrammarMatcher(grammar) for token_id in b'{"key': assert matcher.accept_token(token_id) assert not matcher.accept_token(1) matcher = xgr.GrammarMatcher(grammar) for token_id in b'{"key': assert matcher.accept_token(token_id) assert matcher.accept_token(ord("\\")) def test_utf8_object_array_in_enum(): schema = { "type": "object", "enum": [ {"key": "こんにちは"}, {"key": "😊"}, {"key": "你好"}, {"key": "hello"}, {"key": "\n"}, [123, "こんにちは", "😊", "你好", "hello", "\n"], ], } grammar = xgr.Grammar.from_json_schema(schema) assert _is_grammar_accept_string(grammar, '{"key":"こんにちは"}') assert _is_grammar_accept_string(grammar, '{"key":"😊"}') assert _is_grammar_accept_string(grammar, '{"key":"你好"}') assert _is_grammar_accept_string(grammar, '{"key":"hello"}') assert _is_grammar_accept_string(grammar, '{"key":"\\n"}') assert _is_grammar_accept_string(grammar, '[123,"こんにちは","😊","你好","hello","\\n"]') def test_utf8_object_const(): schema = {"type": "object", "const": {"key": "こんにちは常数constじょうすう\n\r\t"}} grammar = xgr.Grammar.from_json_schema(schema) assert _is_grammar_accept_string(grammar, '{"key":"こんにちは常数constじょうすう\\n\\r\\t"}') def test_utf8_array_const(): schema = {"type": "array", "const": ["こんにちは", "😊", "你好", "hello", "\n"]} grammar = xgr.Grammar.from_json_schema(schema) assert _is_grammar_accept_string(grammar, '["こんにちは","😊","你好","hello","\\n"]') def test_pattern_properties_with_properties(): """Regression test for #487: patternProperties + properties should not ignore properties.""" schema = { "type": "object", "properties": {"name": {"type": "string"}, "grade": {"type": "string"}}, "required": ["name", "grade"], "patternProperties": {"^grade$": {"type": "string"}}, } check_schema_with_instance(schema, {"name": "John", "grade": "B"}, any_whitespace=False) check_schema_with_instance(schema, {"grade": "B"}, is_accepted=False, any_whitespace=False) def test_pattern_properties_extra_key(): """Regression test for #487: patternProperties type constraints must be enforced.""" schema = { "type": "object", "properties": {"name": {"type": "string"}}, "patternProperties": {"^extra_.*$": {"type": "integer"}}, } check_schema_with_instance(schema, {"name": "John", "extra_1": 42}, any_whitespace=False) check_schema_with_instance( schema, {"name": "John", "extra_1": "not_a_number"}, is_accepted=False, any_whitespace=False ) def test_pattern_properties_additional_false(): """Regression test for #487: additionalProperties=false with both properties and patternProperties.""" schema = { "type": "object", "properties": {"name": {"type": "string"}}, "patternProperties": {"^grade$": {"type": "string"}}, "additionalProperties": False, } check_schema_with_instance(schema, {"name": "John"}, any_whitespace=False) check_schema_with_instance(schema, {"name": "John", "grade": "A"}, any_whitespace=False) check_schema_with_instance( schema, {"name": "John", "other": "x"}, is_accepted=False, any_whitespace=False ) def test_property_names_no_trailing_content(): """Regression test for #487: propertyNames with generic pattern must not allow trailing content.""" schema = { "type": "object", "properties": {"name": {"type": "string"}, "grade": {"type": "string"}}, "required": ["name", "grade"], "propertyNames": {"pattern": "^.*$"}, } check_schema_with_instance(schema, {"name": "John", "grade": "B"}, any_whitespace=False) grammar = xgr.Grammar.from_json_schema(json.dumps(schema), any_whitespace=False) assert not _is_grammar_accept_string(grammar, '{"name":"John","grade":"B"} extra') assert not _is_grammar_accept_string(grammar, '{"name":"John","grade":"B"}{}') def test_property_names_with_properties(): """Regression test for #487: propertyNames pattern should constrain key names.""" schema = { "type": "object", "properties": {"name": {"type": "string"}}, "propertyNames": {"pattern": "^[a-z]+$"}, } check_schema_with_instance(schema, {"name": "John"}, any_whitespace=False) check_schema_with_instance(schema, {"Name": "John"}, is_accepted=False, any_whitespace=False) def test_multiple_pattern_properties_with_properties(): """Regression test for #487: multiple patternProperties + properties coexistence.""" schema = { "type": "object", "properties": {"name": {"type": "string"}}, "patternProperties": {"^extra_.*$": {"type": "integer"}, "^meta_.*$": {"type": "string"}}, } check_schema_with_instance( schema, {"name": "John", "extra_1": 42, "meta_tag": "info"}, any_whitespace=False ) check_schema_with_instance(schema, {"name": "John"}, any_whitespace=False) check_schema_with_instance( schema, {"name": "John", "extra_1": "not_int"}, is_accepted=False, any_whitespace=False ) def test_forward_slash_in_const(): # Regression: picojson used to serialize const/enum strings with "\/" # for every "/", which the EBNF lexer then rejected as an invalid # escape sequence (or, in lenient builds, only accepted the escaped # literal form, never the plain JSON the model actually emits). schema = {"const": "http://example.com/path"} grammar = xgr.Grammar.from_json_schema(schema) assert _is_grammar_accept_string(grammar, '"http://example.com/path"') assert not _is_grammar_accept_string(grammar, '"http:\\/\\/example.com\\/path"') def test_forward_slash_in_enum(): schema = {"enum": ["a/b", "c/d/e"]} grammar = xgr.Grammar.from_json_schema(schema) assert _is_grammar_accept_string(grammar, '"a/b"') assert _is_grammar_accept_string(grammar, '"c/d/e"') assert not _is_grammar_accept_string(grammar, '"a\\/b"') def _accept_any_order( schema: Dict[str, Any], instance: str, expect: bool, *, any_order: bool = True, any_whitespace: bool = False, ): grammar = xgr.Grammar.from_json_schema( json.dumps(schema), any_whitespace=any_whitespace, any_order=any_order ) assert _is_grammar_accept_string(grammar, instance) == expect def test_any_order_ebnf(): schema = { "type": "object", "properties": {"a": {"type": "integer"}, "b": {"type": "string"}, "c": {"type": "boolean"}}, "required": ["a", "b"], "additionalProperties": False, } ebnf = _json_schema_to_ebnf(schema, any_whitespace=False, any_order=True) # One "item" alternation repeated [n=#required=2, m=unbounded] times. assert ebnf == basic_json_rules_ebnf_no_space + ( r"""root_item ::= "\"a\"" ": " basic_integer | "\"b\"" ": " basic_string | "\"c\"" ": " basic_boolean root ::= "{" "" (root_item (", " root_item){1,} ) "" "}" """ ) @pytest.mark.parametrize( "instance, expect", [ ('{"a": 1, "b": "x"}', True), # declared order ('{"b": "x", "a": 1}', True), # reordered required ( '{"a": 1, "a": 2}', True, ), # duplicate required, b missing -> only the count (2) is enforced ('{"a": 1, "b": "x", "c": true}', True), # with optional ('{"b": "x", "a": 1, "c": true}', True), # reordered required + optional ('{"c": true, "a": 1, "b": "x"}', True), # optional fully interleaved before required ('{"a": 1, "c": true, "b": "x"}', True), # optional between the two required entries ('{"a": 1}', False), # only one required entry ('{"a": 1, "b": "x", "c": true, "c": false}', True), # other entries are not count-limited ('{"a": 1, "b": "x", "d": 5}', False), # additionalProperties false ("{}", False), # required present -> not empty ], ) def test_any_order_acceptance(instance: str, expect: bool): schema = { "type": "object", "properties": {"a": {"type": "integer"}, "b": {"type": "string"}, "c": {"type": "boolean"}}, "required": ["a", "b"], "additionalProperties": False, } _accept_any_order(schema, instance, expect) def test_any_order_additional_properties_unbounded(): schema = { "type": "object", "properties": {"a": {"type": "integer"}, "b": {"type": "string"}}, "required": ["a"], "additionalProperties": True, } _accept_any_order(schema, '{"a": 1}', True) _accept_any_order(schema, '{"a": 1, "b": "x"}', True) _accept_any_order( schema, '{"a": 1, "z": 5, "y": "q", "w": true}', True ) # extra keys, unbounded def test_any_order_pattern_properties_unbounded(): schema = { "type": "object", "properties": {"a": {"type": "integer"}}, "required": ["a"], "patternProperties": {"^x_": {"type": "integer"}}, "additionalProperties": False, } _accept_any_order(schema, '{"a": 1}', True) _accept_any_order(schema, '{"a": 1, "x_": 5, "x_": 9}', True) # pattern keys, unbounded def test_any_order_applies_to_nested_objects(): schema = { "type": "object", "properties": { "outer_a": {"type": "integer"}, "nested": { "type": "object", "properties": {"x": {"type": "integer"}, "y": {"type": "integer"}}, "required": ["x", "y"], "additionalProperties": False, }, }, "required": ["outer_a", "nested"], "additionalProperties": False, } # any_order applies to every object: both the top-level and the nested object are reorderable. _accept_any_order(schema, '{"nested": {"x": 1, "y": 2}, "outer_a": 5}', True) _accept_any_order(schema, '{"outer_a": 5, "nested": {"y": 2, "x": 1}}', True) _accept_any_order(schema, '{"nested": {"y": 2, "x": 1}, "outer_a": 5}', True) def test_any_order_no_required_fields(): schema = { "type": "object", "properties": {"a": {"type": "integer"}, "b": {"type": "string"}}, "additionalProperties": False, } _accept_any_order(schema, "{}", True) # empty allowed _accept_any_order(schema, '{"b": "x"}', True) _accept_any_order(schema, '{"b": "x", "a": 1}', True) _accept_any_order(schema, '{"a": 1, "a": 2, "b": "x"}', True) # unbounded, no count limit def test_any_order_min_max_properties(): # required {a}, optional {b, c}; total properties must be in [2, 3]. schema = { "type": "object", "properties": {"a": {"type": "integer"}, "b": {"type": "string"}, "c": {"type": "boolean"}}, "required": ["a"], "additionalProperties": False, "minProperties": 2, "maxProperties": 3, } _accept_any_order(schema, '{"a": 1, "b": "x"}', True) # 2 props _accept_any_order(schema, '{"a": 1, "c": true}', True) # 2 props _accept_any_order(schema, '{"a": 1, "b": "x", "c": true}', True) # 3 props _accept_any_order(schema, '{"a": 1, "c": true, "b": "x"}', True) # 3 props, optional reordered _accept_any_order(schema, '{"a": 1}', False) # 1 prop < minProperties=2 def test_any_order_max_properties_equals_required_count(): # maxProperties == #required (2): no room for the optional field. schema = { "type": "object", "properties": {"a": {"type": "integer"}, "b": {"type": "string"}, "c": {"type": "boolean"}}, "required": ["a", "b"], "additionalProperties": False, "maxProperties": 2, } _accept_any_order(schema, '{"a": 1, "b": "x"}', True) # exactly the 2 required _accept_any_order(schema, '{"b": "x", "a": 1}', True) # reordered required _accept_any_order( schema, '{"a": 1, "b": "x", "c": true}', False ) # 3 props > max=2, no optional allowed def test_any_order_min_properties_with_additional(): # required {a}; additionalProperties allowed; minProperties=3 => >= 2 optional/extra entries. schema = { "type": "object", "properties": {"a": {"type": "integer"}, "b": {"type": "string"}}, "required": ["a"], "additionalProperties": True, "minProperties": 3, } _accept_any_order(schema, '{"a": 1, "b": "x"}', False) # 2 props < min=3 _accept_any_order(schema, '{"a": 1, "b": "x", "z": 5}', True) # 3 props (extra key) _accept_any_order(schema, '{"a": 1, "z": 5, "y": 6, "w": 7}', True) # 4 props, unbounded above def test_any_order_backward_compatible(): schema = { "type": "object", "properties": {"a": {"type": "integer"}, "b": {"type": "string"}}, "required": ["a", "b"], "additionalProperties": False, } # any_order=False (the default) must produce the exact same grammar as before. default = _json_schema_to_ebnf(schema, any_whitespace=False) explicit_false = _json_schema_to_ebnf(schema, any_whitespace=False, any_order=False) assert default == explicit_false # The any_order-only "item" alternation rule must not appear in the fixed-order grammar. assert "root_item" not in default assert "root_item" in _json_schema_to_ebnf(schema, any_whitespace=False, any_order=True) def test_any_order_qwen_xml(): from xgrammar.testing import _qwen_xml_tool_calling_to_ebnf schema = { "type": "object", "properties": {"a": {"type": "integer"}, "b": {"type": "string"}}, "required": ["a", "b"], "additionalProperties": False, } ordered = _qwen_xml_tool_calling_to_ebnf(json.dumps(schema), False) any_order = _qwen_xml_tool_calling_to_ebnf(json.dumps(schema), True) # Both grammars share the same basic_*/xml_* prefix; only the root rules differ. prefix = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" xml_string ::= TagDispatch(loop_after_dispatch=false,excludes=("")) xml_any ::= xml_string | basic_array | basic_object xml_object ::= ( [ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "" ([ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "")* [ \n\t]*) | [ \n\t]* xml_variable_name ::= [a-zA-Z_][a-zA-Z0-9_]* root_prop_0 ::= ("0" | "-"? [1-9] [0-9]*) """ # Ordered: the required props are emitted in fixed declared order (a, then b). assert ordered == prefix + ( r"""root_part_0 ::= [ \n\t]* "" [ \n\t]* xml_string [ \n\t]* "" "" root ::= [ \n\t]* (("" [ \n\t]* root_prop_0 [ \n\t]* "" root_part_0)) [ \n\t]* """ ) # any_order: one "item" alternation repeated [n=#required=2, m=unbounded] times. assert any_order == prefix + ( r"""root_item ::= "" [ \n\t]* root_prop_0 [ \n\t]* "" | "" [ \n\t]* xml_string [ \n\t]* "" root ::= [ \n\t]* (root_item ([ \n\t]* root_item){1,} ) [ \n\t]* """ ) @pytest.mark.parametrize("cache_enabled", [True, False]) def test_compile_json_schema_any_order(cache_enabled: bool): # Regression: compile_json_schema once dropped any_order on the value-producing path, silently # returning a fixed-order grammar. Both cache states route through that call. tokenizer_info = xgr.TokenizerInfo([f"<{i}>" for i in range(16)]) compiler = xgr.GrammarCompiler(tokenizer_info, cache_enabled=cache_enabled) schema = { "type": "object", "properties": {"a": {"type": "integer"}, "b": {"type": "string"}}, "required": ["a", "b"], "additionalProperties": False, } ordered = str(compiler.compile_json_schema(schema, any_whitespace=False).grammar) any_order = str( compiler.compile_json_schema(schema, any_whitespace=False, any_order=True).grammar ) # any_order=True relaxes ordering via the flat "item" alternation; the default does not. assert "root_item" not in ordered assert "root_item" in any_order assert ordered != any_order if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.2.3/tests/python/test_recursion_depth.py000066400000000000000000000031201521764210300226200ustar00rootroot00000000000000import sys import pytest import xgrammar as xgr from xgrammar.testing import _get_matcher_from_grammar @pytest.mark.thread_unsafe def test_set_get_recursion_depth(): """Test getting default recursion depth""" default_depth = xgr.get_max_recursion_depth() assert default_depth == 10000 xgr.set_max_recursion_depth(1000) new_depth = xgr.get_max_recursion_depth() assert new_depth == 1000 xgr.set_max_recursion_depth(default_depth) @pytest.mark.thread_unsafe def test_recursion_depth_context(): """Test recursion depth context manager""" assert xgr.get_max_recursion_depth() == 10000 with xgr.max_recursion_depth(1000): depth = xgr.get_max_recursion_depth() assert depth == 1000 assert xgr.get_max_recursion_depth() == 10000 def test_error_set_recursion_depth(): """Test setting recursion depth to an invalid value""" with pytest.raises(RuntimeError): xgr.set_max_recursion_depth(-1) with pytest.raises(RuntimeError): xgr.set_max_recursion_depth(100000000) def test_recursion_exceed(): # In Earley Parser, the recursion depth can't be exceeded. with xgr.max_recursion_depth(1000): grammar_ebnf = r""" root ::= "\"" basic_string "\"" basic_string ::= "" | [^"\\\r\n] basic_string | "\\" escape basic_string escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] """ input_str = '"' + " " * 10000 + '"' matcher = _get_matcher_from_grammar(grammar_ebnf) matcher.accept_string(input_str) if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.2.3/tests/python/test_regex_converter.py000066400000000000000000000416301521764210300226340ustar00rootroot00000000000000import sys import time import pytest from transformers import AutoTokenizer import xgrammar as xgr from xgrammar.testing import _is_grammar_accept_string, _regex_to_ebnf def test_basic(): regex = "123" grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= "1" "2" "3" """ assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, "123") assert not _is_grammar_accept_string(grammar_str, "1234") def test_unicode(): regex = "ww我😁" grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= "w" "w" "\u6211" "\U0001f601" """ assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, regex) regex_expected_grammar_instance = [ ( r"\^\$\.\*\+\?\\\(\)\[\]\{\}\|\/", r"""root ::= "^" "$" "." "*" "+" "\?" "\\" "(" ")" "[" "]" "{" "}" "|" "/" """, "^$.*+?\\()[]{}|/", ), ( r"\"\'\a\f\n\r\t\v\0\e", r"""root ::= "\"" "\'" "\a" "\f" "\n" "\r" "\t" "\v" "\0" "\e" """, "\"'\a\f\n\r\t\v\0\x1b", ), ( r"\u{20BB7}\u0300\x1F\cJ", r"""root ::= "\U00020bb7" "\u0300" "\x1f" "\n" """, "\U00020bb7\u0300\x1f\n", ), ( r"[\r\n\$\u0010-\u006F\]\--]+", r"""root ::= [\r\n$\x10-o\]\--]+ """, "\r\n$\u0020-", # TODO(yixin): add unicode tests ), ] @pytest.mark.parametrize("regex, expected_grammar, instance", regex_expected_grammar_instance) def test_escape(regex: str, expected_grammar: str, instance: str): grammar_str = _regex_to_ebnf(regex) assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) def test_escaped_char_class(): # TODO(yixin): add unicode tests # TODO(yixin): add tests for escaped char class nested in char class regex = r"\w\w\W\d\D\s\S" instance = "A_ 1b 0" grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= [a-zA-Z0-9_] [a-zA-Z0-9_] [^a-zA-Z0-9_] [0-9] [^0-9] [\f\n\r\t\v\u0020\u00a0] [^[\f\n\r\t\v\u0020\u00a0] """ assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) def test_char_class(): regex = r"[-a-zA-Z+--]+" instance = "a-+" grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= [-a-zA-Z+--]+ """ assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) def test_boundary(): regex = r"^abc$" instance = "abc" grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= "a" "b" "c" """ assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) def test_disjunction(): regex = r"abc|de(f|g)" instance = "deg" grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= "a" "b" "c" | "d" "e" ( "f" | "g" ) """ assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) def test_space(): regex = r" abc | df | g " instance = " df " grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= " " "a" "b" "c" " " | " " "d" "f" " " | " " "g" " " """ assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) def test_quantifier(): regex = r"(a|b)?[a-z]+(abc)*" instance = "adddabcabc" instance1 = "z" grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= ( "a" | "b" )? [a-z]+ ( "a" "b" "c" )* """ # TODO(yixin): add tests for repetition range assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) assert _is_grammar_accept_string(grammar_str, instance1) def test_consecutive_quantifiers(): regex = "a{1,3}?{1,3}" with pytest.raises(RuntimeError, match="Two consecutive repetition modifiers are not allowed."): _regex_to_ebnf(regex) regex = "a???" with pytest.raises(RuntimeError, match="Two consecutive repetition modifiers are not allowed."): _regex_to_ebnf(regex) regex = "a++" with pytest.raises(RuntimeError, match="Two consecutive repetition modifiers are not allowed."): _regex_to_ebnf(regex) regex = "a+?{1,3}" with pytest.raises(RuntimeError, match="Two consecutive repetition modifiers are not allowed."): _regex_to_ebnf(regex) def test_group(): regex = r"(a|b)(c|d)" instance = "ac" grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= ( "a" | "b" ) ( "c" | "d" ) """ assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) def test_any(): regex = r".+a.+" instance = "bbbabb" grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= [\u0000-\U0010FFFF]+ "a" [\u0000-\U0010FFFF]+ """ assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) def test_ipv4(): regex = r"((25[0-5]|2[0-4]\d|[01]?\d\d?).)((25[0-5]|2[0-4]\d|[01]?\d\d?).)((25[0-5]|2[0-4]\d|[01]?\d\d?).)(25[0-5]|2[0-4]\d|[01]?\d\d?)" grammar_str = _regex_to_ebnf(regex) expected_grammar = ( r"""root ::= ( ( "2" "5" [0-5] | "2" [0-4] [0-9] | [01]? [0-9] [0-9]? ) """ r"""[\u0000-\U0010FFFF] ) ( ( "2" "5" [0-5] | "2" [0-4] [0-9] | [01]? [0-9] """ r"""[0-9]? ) [\u0000-\U0010FFFF] ) ( ( "2" "5" [0-5] | "2" [0-4] [0-9] | [01]? [0-9] """ r"""[0-9]? ) [\u0000-\U0010FFFF] ) ( "2" "5" [0-5] | "2" [0-4] [0-9] | [01]? [0-9] [0-9]? ) """ ) assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, "123.45.67.89") date_time_instances_accepted = [ ("2024-05-19T14:23:45Z", True), ("2019-11-30T08:15:27+05:30", True), ("2030-02-01T22:59:59-07:00", True), ("2021-07-04T00:00:00.123456Z", True), ("2022-12-31T23:45:12-03:00", True), ("2024-13-15T14:30:00Z", False), ("2023-02-2010:59:59Z", False), ("2021-11-05T24:00:00+05:30", False), ("2022-08-20T12:61:10-03:00", False), ] @pytest.mark.parametrize("instance, accepted", date_time_instances_accepted) def test_date_time(instance: str, accepted: bool): regex = r"^\d\d\d\d-(0[1-9]|1[0-2])-([0-2]\d|3[01])T([01]\d|2[0123]):[0-5]\d:[0-5]\d(\.\d+)?(Z|[+-]([01]\d|2[0123]):[0-5]\d)$" grammar_str = _regex_to_ebnf(regex) expected_grammar = ( r"""root ::= [0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" ( [0-2] [0-9] """ r"""| "3" [01] ) "T" ( [01] [0-9] | "2" [0123] ) ":" [0-5] [0-9] ":" [0-5] [0-9] """ r"""( "." [0-9]+ )? ( "Z" | [+-] ( [01] [0-9] | "2" [0123] ) ":" [0-5] [0-9] ) """ ) assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) == accepted date_instances_accepted = [ ("0024-05-19", True), ("2019-11-30", True), ("2022-12-31", True), ("2024-13-15", False), ("2024-12-32", False), ] @pytest.mark.parametrize("instance, accepted", date_instances_accepted) def test_date(instance: str, accepted: bool): regex = r"^\d\d\d\d-(0[1-9]|1[0-2])-([0-2]\d|3[01])$" grammar_str = _regex_to_ebnf(regex) expected_grammar = ( r"""root ::= [0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" """ r"""( [0-2] [0-9] | "3" [01] ) """ ) assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) == accepted time_instances_accepted = [ ("14:23:45Z", True), ("08:15:27+05:30", True), ("22:59:59-07:00", True), ("00:00:00.123456Z", True), ("10:59:59ZA", False), ("24:00:00+05:30", False), ("12:15:10-03:60", False), ] @pytest.mark.parametrize("instance, accepted", time_instances_accepted) def test_time(instance: str, accepted: bool): regex = r"^([01]\d|2[0123]):[0-5]\d:[0-5]\d(\.\d+)?(Z|[+-]([01]\d|2[0123]):[0-5]\d)$" grammar_str = _regex_to_ebnf(regex) expected_grammar = ( r"""root ::= ( [01] [0-9] | "2" [0123] ) ":" [0-5] [0-9] ":" [0-5] [0-9] """ r"""( "." [0-9]+ )? ( "Z" | [+-] ( [01] [0-9] | "2" [0123] ) ":" [0-5] [0-9] ) """ ) assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) == accepted email_instances_accepted = [ ("simple@example.com", True), ("very.common@example.com", True), ("user_name+123@example.co.uk", True), ('"john.doe"@example.org', True), ("mail-host@online-shop.biz", True), ("customer/department=shipping@example.com", True), ("$A12345@example.non-profit.org", True), ('"!def!xyz%abc"@example.com', True), ("support@192.168.1.1", True), ("plainaddress", False), ("@missingusername.com", False), ("user@.com.my", False), ("user@com", False), ("user@-example.com", False), ] @pytest.mark.parametrize("instance, accepted", email_instances_accepted) def test_email(instance: str, accepted: bool): regex = ( r"""^([\w!#$%&'*+/=?^_`{|}~-]+(\.[\w!#$%&'*+/=?^_`{|}~-]+)*""" r"""|"([\w!#$%&'*+/=?^_`{|}~\-(),:;<>@[\].]|\\")+")@(([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+""" r"""[a-z0-9]([a-z0-9-]*[a-z0-9])?)$""" ) grammar_str = _regex_to_ebnf(regex) assert _is_grammar_accept_string(grammar_str, instance) == accepted def test_empty_character_class(): regex = "[]" with pytest.raises(RuntimeError, match="Empty character class is not allowed in regex."): _regex_to_ebnf(regex) def test_group_modifiers(): # Test non-capturing group regex = "(?:abc)" grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= ( "a" "b" "c" ) """ assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, "abc") # Test named capturing group regex = "(?abc)" grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= ( "a" "b" "c" ) """ assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, "abc") # Test unsupported group modifiers unsupported_regexes = [ "(?=abc)", # Positive lookahead "(?!abc)", # Negative lookahead "(?<=abc)", # Positive lookbehind "(?@[\].]|\\")+")@(([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+""" r"""[a-z0-9]([a-z0-9-]*[a-z0-9])?)$""" ), "customer/department=shipping@test.example.test-example.com", ), ] tokenizer_path_regex_instance = [(t, *ri) for t in tokenizer_paths for ri in regex_instances] @pytest.mark.hf_token_required @pytest.mark.parametrize("tokenizer_path, regex, instance", tokenizer_path_regex_instance) def test_mask_generation(tokenizer_path: str, regex: str, instance: str): print(f"Tokenizer: {tokenizer_path}, regex: {regex}, instance: {instance}") tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) grammar_compiler = xgr.GrammarCompiler(tokenizer_info, cache_enabled=False) time_start = time.monotonic_ns() matcher_compiled_grammar = grammar_compiler.compile_grammar(_regex_to_ebnf(regex)) time_end = time.monotonic_ns() print(f"Time for preprocessing: {(time_end - time_start) / 1e3} us") matcher = xgr.GrammarMatcher(matcher_compiled_grammar) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) for c in instance.encode("utf-8"): time_start = time.monotonic_ns() matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time for fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") accepted = matcher.accept_string(bytes([c])) assert accepted print(f"Accepting {c}") time_start = time.monotonic_ns() matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time for fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") assert matcher.accept_token(tokenizer.eos_token_id) assert matcher.is_terminated() empty_regex = ["", "^$", "(())", "()", "^", "$", "()|()"] @pytest.mark.parametrize("regex", empty_regex) def test_empty(regex: str): grammar = xgr.Grammar.from_regex(regex) expected_grammar = 'root ::= ("")\n' assert str(grammar) == expected_grammar assert _is_grammar_accept_string(grammar, "") assert not _is_grammar_accept_string(grammar, "a") if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.2.3/tests/python/test_serialization.py000066400000000000000000000374641521764210300223220ustar00rootroot00000000000000# -*- coding: utf-8 -*- import json import subprocess import sys import textwrap from typing import Any, List, Tuple import pytest from pydantic import BaseModel, RootModel from transformers import AutoTokenizer # type: ignore import xgrammar as xgr from xgrammar.testing import _is_grammar_accept_string def construct_grammar(): """Construct a Grammar object for testing.""" return xgr.Grammar.from_ebnf( """rule1 ::= ([^0-9] rule1) | "" root_rule ::= rule1 "a" """, root_rule_name="root_rule", ) def construct_tokenizer_info(): """Construct a TokenizerInfo object for testing.""" return xgr.TokenizerInfo( ["1", "212", "a", "A", "b", "一", "-", "aBc", "abc"], vocab_type=xgr.VocabType.BYTE_FALLBACK, vocab_size=10, stop_token_ids=[0, 1], add_prefix_space=True, ) def construct_compiled_grammar(): """Construct a CompiledGrammar object for testing.""" tokenizer_info = construct_tokenizer_info() grammar = construct_grammar() grammar_compiler = xgr.GrammarCompiler(tokenizer_info) return grammar_compiler.compile_grammar(grammar), tokenizer_info def test_get_serialization_version(): """Test the version of the serialized JSON string.""" assert xgr.get_serialization_version() == "v14" def test_serialize_grammar(): """Test Grammar serialization produces expected JSON string.""" grammar = construct_grammar() serialized = grammar.serialize_json() expected_json = { "rules": [["rule1", 4, -1, False], ["root", 8, -1, False]], "grammar_expr_data": [0, 5, 8, 12, 14, 18, 21, 24, 28], "grammar_expr_indptr": [ # fmt: off 1,3,1,48,57,4,1,0,5,2,0,1,3,0,6,2,3,2,4,1,0,0,1,97,5,2,5,6,6,1,7 # fmt: on ], "root_rule_id": 1, "complete_fsm": None, "per_rule_fsms": [], "allow_empty_rule_ids": [], "optimized": False, "__VERSION__": "v14", } # The fsms are the same one, but the start state and end states are different. assert json.loads(serialized) == expected_json def test_serialize_grammar_exception(): """Test Grammar serialization produces expected JSON string.""" expected_json = { "rules": [["rule1", 4, 9, True], ["root", 8, -1, False]], "grammar_expr_data": [0, 2, 7, 10, 14, 18, 21, 24, 28, 31], "grammar_expr_indptr": [ # fmt: off 3,0,1,3,1,48,57,4,1,0,5,2,1,2,6,2,0,3,4,1,0,0,1,97,5,2,5,6,6,1,7,5,1,6 # fmt: on ], "root_rule_id": 1, "allow_empty_rule_ids": [], "complete_fsm": None, "per_rule_fsms": [], "__VERSION__": "v14", } expected_json["__VERSION__"] = "v1" # Change version to trigger error with pytest.raises(xgr.DeserializeVersionError): xgr.Grammar.deserialize_json(json.dumps(expected_json)) expected_json["__VERSION__"] = "v14" expected_json.pop("rules") # Remove required field to trigger error with pytest.raises(xgr.DeserializeFormatError): xgr.Grammar.deserialize_json(json.dumps(expected_json)) with pytest.raises(xgr.InvalidJSONError): xgr.Grammar.deserialize_json("not a valid json string") def test_serialize_grammar_roundtrip(): """Test Grammar serialization and deserialization roundtrip.""" original_grammar = construct_grammar() serialized = original_grammar.serialize_json() recovered_grammar = xgr.Grammar.deserialize_json(serialized) serialized_new = recovered_grammar.serialize_json() assert serialized == serialized_new def test_serialize_grammar_functional(): """Test that deserialized Grammar object functions correctly.""" original_grammar = construct_grammar() serialized = original_grammar.serialize_json() recovered_grammar = xgr.Grammar.deserialize_json(serialized) # Test functional equivalence by checking string representation assert str(original_grammar) == str(recovered_grammar) # Test with GrammarMatcher functionality tokenizer_info = construct_tokenizer_info() compiler = xgr.GrammarCompiler(tokenizer_info) compiled_original = compiler.compile_grammar(original_grammar) compiled_recovered = compiler.compile_grammar(recovered_grammar) matcher_original = xgr.GrammarMatcher(compiled_original) matcher_recovered = xgr.GrammarMatcher(compiled_recovered) # Test that both matchers accept the same input test_input = "aaa" assert matcher_original.accept_string(test_input) == matcher_recovered.accept_string(test_input) def test_serialize_tokenizer_info(): """Test TokenizerInfo serialization produces expected JSON string.""" tokenizer_info = construct_tokenizer_info() serialized = tokenizer_info.serialize_json() expected_json = ( '{"vocab_type":1,"vocab_size":10,"add_prefix_space":true,' '"stop_token_ids":[0,1],"special_token_ids":[9],' '"decoded_vocab":["1","212","a","A","b","\\u00e4\\u00b8\\u0080","-","aBc","abc"],' '"sorted_decoded_vocab":[[6,"-"],[3,"A"],[2,"a"],[7,"aBc"],[8,"abc"],[4,"b"],[5,"\\u00e4\\u00b8\\u0080"]],' '"trie_subtree_nodes_range":[1,2,5,4,5,6,7],' '"__VERSION__":"v14"}' ) assert json.loads(serialized) == json.loads(expected_json) def test_serialize_tokenizer_info_roundtrip(): """Test TokenizerInfo serialization and deserialization roundtrip.""" original_tokenizer_info = construct_tokenizer_info() serialized = original_tokenizer_info.serialize_json() recovered_tokenizer_info = xgr.TokenizerInfo.deserialize_json(serialized) serialized_new = recovered_tokenizer_info.serialize_json() assert serialized == serialized_new def test_serialize_tokenizer_info_functional(): """Test that deserialized TokenizerInfo object functions correctly.""" original_tokenizer_info = construct_tokenizer_info() serialized = original_tokenizer_info.serialize_json() recovered_tokenizer_info = xgr.TokenizerInfo.deserialize_json(serialized) # Test property equivalence assert original_tokenizer_info.vocab_type == recovered_tokenizer_info.vocab_type assert original_tokenizer_info.vocab_size == recovered_tokenizer_info.vocab_size assert original_tokenizer_info.add_prefix_space == recovered_tokenizer_info.add_prefix_space assert original_tokenizer_info.stop_token_ids == recovered_tokenizer_info.stop_token_ids assert original_tokenizer_info.special_token_ids == recovered_tokenizer_info.special_token_ids assert original_tokenizer_info.decoded_vocab == recovered_tokenizer_info.decoded_vocab # Test functional equivalence with GrammarCompiler grammar = construct_grammar() compiler_original = xgr.GrammarCompiler(original_tokenizer_info) compiler_recovered = xgr.GrammarCompiler(recovered_tokenizer_info) compiled_original = compiler_original.compile_grammar(grammar) compiled_recovered = compiler_recovered.compile_grammar(grammar) # Both should produce functional matchers matcher_original = xgr.GrammarMatcher(compiled_original) matcher_recovered = xgr.GrammarMatcher(compiled_recovered) test_input = "aaa" assert matcher_original.accept_string(test_input) == matcher_recovered.accept_string(test_input) def test_serialize_compiled_grammar(): """Test CompiledGrammar serialization produces expected JSON string. We verify the adaptive token mask part separately. """ compiled_grammar, tokenizer_info = construct_compiled_grammar() serialized = compiled_grammar.serialize_json() expected_json = { "grammar": { "rules": [["rule1", 4, 9, True], ["root", 8, -1, False]], "grammar_expr_data": [0, 2, 7, 10, 14, 18, 21, 24, 28, 31], "grammar_expr_indptr": [ # fmt: off 3,0,1,3,1,48,57,4,1,0,5,2,1,2,6,2,0,3,4,1,0,0,1,97,5,2,5,6,6,1,7,5,1,6 # fmt: on ], "root_rule_id": 1, "allow_empty_rule_ids": [0], # fmt: off "complete_fsm": { "edges": { "data_": [[128, 191, 1], [128, 191, 3], [0, 47, 3], [58, 127, 3], [192, 223, 1],[224, 239, 0],[240, 247, 4],[-2, 0, 5], [128, 191, 0],[97, 97, 8],[-2, 0, 6]], "indptr_":[0, 1, 2, 7, 8, 9, 9, 10, 11, 11] }, "edge_aux_data": [], "edge_num": 11, }, "per_rule_fsms": [ [ [ { "edges": { "data_": [[128, 191, 1], [128, 191, 3], [0, 47, 3], [58, 127, 3], [192, 223, 1],[224, 239, 0],[240, 247, 4],[-2, 0, 5], [128, 191, 0],[97, 97, 8],[-2, 0, 6]], "indptr_":[0, 1, 2, 7, 8, 9, 9, 10, 11, 11] }, "edge_aux_data": [], "edge_num": 11, }, 2, [2, 5], False, 11 ], 9, 6 ], [ [ { "edges": { "data_": [[128, 191, 1], [128, 191, 3], [0, 47, 3], [58, 127, 3], [192, 223, 1],[224, 239, 0],[240, 247, 4],[-2, 0, 5], [128, 191, 0],[97, 97, 8],[-2, 0, 6]], "indptr_":[0, 1, 2, 7, 8, 9, 9, 10, 11, 11] }, "edge_aux_data": [], "edge_num": 11, }, 7, [8], False, 11, ], 2, 3 ] ], # fmt: on "optimized": True, }, "tokenizer_metadata": { "vocab_type": 1, "vocab_size": 10, "add_prefix_space": True, "stop_token_ids": [0, 1], }, "__VERSION__": "v14", } class AdaptiveTokenMask(BaseModel): store_type: int accepted_indices: List[int] rejected_indices: List[int] accepted_bitset: Any uncertain_indices: List[int] class AdaptiveTokenMaskCache(RootModel): root: List[Tuple[List[int], AdaptiveTokenMask]] recovered_obj = json.loads(serialized) adaptive_token_mask_cache = recovered_obj.pop("adaptive_token_mask_cache", None) print(serialized) assert recovered_obj == expected_json AdaptiveTokenMaskCache.model_validate(adaptive_token_mask_cache) def test_serialize_compiled_grammar_roundtrip(): """Test CompiledGrammar serialization and deserialization roundtrip.""" original_compiled_grammar, tokenizer_info = construct_compiled_grammar() serialized = original_compiled_grammar.serialize_json() recovered_compiled_grammar = xgr.CompiledGrammar.deserialize_json(serialized, tokenizer_info) serialized_new = recovered_compiled_grammar.serialize_json() assert serialized == serialized_new def test_serialize_compiled_grammar_functional(): """Test that deserialized CompiledGrammar object functions correctly.""" original_compiled_grammar, tokenizer_info = construct_compiled_grammar() serialized = original_compiled_grammar.serialize_json() recovered_compiled_grammar = xgr.CompiledGrammar.deserialize_json(serialized, tokenizer_info) # Test that both create functional matchers matcher_original = xgr.GrammarMatcher(original_compiled_grammar) matcher_recovered = xgr.GrammarMatcher(recovered_compiled_grammar) # Test token mask generation token_bitmask_original = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) token_bitmask_recovered = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) # Both should generate the same masks assert matcher_original.fill_next_token_bitmask( token_bitmask_original ) == matcher_recovered.fill_next_token_bitmask(token_bitmask_recovered) # Import torch for tensor comparison import torch torch.testing.assert_close(token_bitmask_original, token_bitmask_recovered) # Test input acceptance test_input = "aaa" assert matcher_original.accept_string(test_input) == matcher_recovered.accept_string(test_input) assert matcher_original.is_terminated() == matcher_recovered.is_terminated() @pytest.mark.hf_token_required def test_serialize_compiled_grammar_with_hf_tokenizer(): """Test CompiledGrammar serialization with a real HuggingFace tokenizer.""" tokenizer = AutoTokenizer.from_pretrained( "meta-llama/Llama-3.1-8B-Instruct", use_fast=True, trust_remote_code=True ) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) grammar_compiler = xgr.GrammarCompiler(tokenizer_info) # Test with JSON schema class TestModel(BaseModel): name: str age: int # Compile grammar compiled_grammar = grammar_compiler.compile_json_schema(TestModel) # Serialize and deserialize tokenizer_info_json = tokenizer_info.serialize_json() tokenizer_info_recovered = xgr.TokenizerInfo.deserialize_json(tokenizer_info_json) serialized = compiled_grammar.serialize_json() recovered_compiled_grammar = xgr.CompiledGrammar.deserialize_json( serialized, tokenizer_info_recovered ) # Test functional equivalence test_json = '{"name": "John", "age": 30}' token_ids = tokenizer.encode(test_json)[1:] # skip the initial BOS token matcher = xgr.GrammarMatcher(recovered_compiled_grammar) bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) for token_id in token_ids: matcher.fill_next_token_bitmask(bitmask) masked_token_ids = xgr.testing._get_masked_tokens_from_bitmask( bitmask, tokenizer_info.vocab_size ) assert token_id not in masked_token_ids assert matcher.accept_token(token_id) assert matcher.accept_token(tokenizer.eos_token_id) assert matcher.is_terminated() def test_serialize_grammar_utf8(): """Test Grammar serialization with UTF-8 characters.""" grammar = xgr.Grammar.from_ebnf('root ::= "こんにちは" | "😊" | "你好" | "hello" | "\\n"') serialized = grammar.serialize_json() recovered_grammar = xgr.Grammar.deserialize_json(serialized) assert _is_grammar_accept_string(recovered_grammar, "こんにちは") assert _is_grammar_accept_string(recovered_grammar, "😊") assert _is_grammar_accept_string(recovered_grammar, "你好") assert _is_grammar_accept_string(recovered_grammar, "hello") assert _is_grammar_accept_string(recovered_grammar, "\n") def test_serialized_output_survives_pickle(): """Pickling serialized output must not crash (regression for the diskcache segfault). Run in a subprocess so a regression shows up as a non-zero return code instead of segfaulting the whole pytest session. """ script = textwrap.dedent( """ import pickle import xgrammar as xgr grammar = xgr.Grammar.from_ebnf('root ::= "a"') tokenizer_info = xgr.TokenizerInfo( ["a", "b"], vocab_type=xgr.VocabType.BYTE_FALLBACK, vocab_size=2 ) compiled_grammar = xgr.GrammarCompiler(tokenizer_info).compile_grammar(grammar) values = [ xgr.get_serialization_version(), grammar.serialize_json(), tokenizer_info.serialize_json(), compiled_grammar.serialize_json(), ] for value in values: assert pickle.loads(pickle.dumps(value)) == value print("PICKLE_OK") """ ) result = subprocess.run( [sys.executable, "-c", script], capture_output=True, text=True, timeout=300 ) assert result.returncode == 0, f"returncode={result.returncode}\n{result.stderr[-2000:]}" assert "PICKLE_OK" in result.stdout if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.2.3/tests/python/test_speculative_decoding.py000066400000000000000000000226621521764210300236170ustar00rootroot00000000000000"""Test the speculative decoding utilities.""" import sys import pytest import torch import xgrammar as xgr from xgrammar.matcher import allocate_token_bitmask from xgrammar.testing import _traverse_draft_tree VOCAB = ["a", "b", "c", "{", "}", '"', ":", ",", " ", "true", "false", "null"] VOCAB_SIZE = len(VOCAB) # ── Tree definitions ────────────────────────────────────────────────────────── # Linear tree: 0 -> 1 -> 2 LINEAR_TREE = ( torch.tensor([1, 2, -1], dtype=torch.int64), # next_token torch.tensor([-1, -1, -1], dtype=torch.int64), # next_sibling torch.tensor([3, 6, 4], dtype=torch.int64), # draft tokens: {, :, } ) # Tree with siblings: # 0 # / \ # 1 2 SIBLING_TREE = ( torch.tensor([1, -1, -1], dtype=torch.int64), # next_token torch.tensor([-1, 2, -1], dtype=torch.int64), # next_sibling torch.tensor([3, 5, 4], dtype=torch.int64), # draft tokens: {, ", } ) @pytest.fixture(scope="module") def compiled_grammar(): """Compile the built-in JSON grammar with our small test vocab.""" grammar = xgr.Grammar.builtin_json_grammar() tokenizer_info = xgr.TokenizerInfo(VOCAB, vocab_size=VOCAB_SIZE, stop_token_ids=[]) compiler = xgr.GrammarCompiler(tokenizer_info) return compiler.compile_grammar(grammar) def _run_traverse(compiled_grammar, tree, **traverse_kwargs): """Run traverse_draft_tree on a tree and return (result, bitmask).""" retrieve_next_token, retrieve_next_sibling, draft_tokens = tree num_nodes = retrieve_next_token.shape[0] matcher = xgr.GrammarMatcher(compiled_grammar) bitmask = allocate_token_bitmask(num_nodes, VOCAB_SIZE) result = matcher.traverse_draft_tree( retrieve_next_token, retrieve_next_sibling, draft_tokens, bitmask, **traverse_kwargs ) return result, bitmask # ── Basic traversal tests ──────────────────────────────────────────────────── def test_traverse_draft_tree_linear(compiled_grammar): """Test traverse_draft_tree with a simple linear tree structure.""" result, bitmask = _run_traverse(compiled_grammar, LINEAR_TREE) assert result is True assert bitmask[0].any(), "First position bitmask should be non-zero" def test_traverse_draft_tree_with_siblings(compiled_grammar): """Test traverse_draft_tree with a tree that has sibling nodes.""" result, bitmask = _run_traverse(compiled_grammar, SIBLING_TREE) assert result is True assert bitmask[0].any(), "Root position bitmask should be non-zero" def test_traverse_draft_tree_rejected_node(compiled_grammar): """Rejected nodes should have zero bitmasks.""" rejected_tree = ( torch.tensor([1, 2, -1], dtype=torch.int64), torch.tensor([-1, -1, -1], dtype=torch.int64), torch.tensor([3, 0, 3], dtype=torch.int64), ) result, bitmask = _run_traverse(compiled_grammar, rejected_tree) assert result is True assert bitmask[0].any(), "Root position bitmask should be non-zero" assert not bitmask[1].any(), "Rejected node bitmask should be zero" def test_traverse_draft_tree_invalid_token_with_sibling(compiled_grammar): """Invalid draft tokens should not break traversal of valid sibling branches.""" invalid_token_tree = ( torch.tensor([1, 2, -1, -1], dtype=torch.int64), torch.tensor([-1, 3, -1, -1], dtype=torch.int64), torch.tensor([3, 100, 3, 3], dtype=torch.int64), ) result, bitmask = _run_traverse(compiled_grammar, invalid_token_tree) assert result is True assert bitmask[0].any(), "Root position bitmask should be non-zero" assert not bitmask[1].any(), "Invalid token node bitmask should be zero" assert bitmask[3].any(), "Valid sibling bitmask should be computed" def test_traverse_draft_tree_terminated_node(): """Terminated nodes should have zero bitmasks.""" grammar = xgr.Grammar.from_ebnf('root ::= "a"') tokenizer_info = xgr.TokenizerInfo(["a", "b"], vocab_size=2, stop_token_ids=[]) compiler = xgr.GrammarCompiler(tokenizer_info) compiled_grammar = compiler.compile_grammar(grammar) matcher = xgr.GrammarMatcher(compiled_grammar, terminate_without_stop_token=True) bitmask = allocate_token_bitmask(3, 2) terminated_tree = ( torch.tensor([1, 2, -1], dtype=torch.int64), torch.tensor([-1, -1, -1], dtype=torch.int64), torch.tensor([0, 0, 1], dtype=torch.int64), ) result = matcher.traverse_draft_tree(*terminated_tree, bitmask) assert result is True assert bitmask[0].any(), "Root position bitmask should be non-zero" assert not bitmask[1].any(), "Terminated node bitmask should be zero" def test_old_traverse_draft_tree(compiled_grammar): """Test the backward-compatible testing wrapper.""" retrieve_next_token, retrieve_next_sibling, draft_tokens = LINEAR_TREE matcher = xgr.GrammarMatcher(compiled_grammar) bitmask = allocate_token_bitmask(retrieve_next_token.shape[0], VOCAB_SIZE) result = _traverse_draft_tree( retrieve_next_token, retrieve_next_sibling, draft_tokens, matcher, bitmask ) assert result is True assert bitmask[0].any(), "First position bitmask should be non-zero" # ── Shape / dtype validation ───────────────────────────────────────────────── def test_traverse_draft_tree_shape_assertion(compiled_grammar): """Test that traverse_draft_tree raises RuntimeError for mismatched shapes/dtypes.""" matcher = xgr.GrammarMatcher(compiled_grammar) retrieve_next_token = torch.tensor([1, 2, -1], dtype=torch.int64) draft_tokens = torch.tensor([3, 6, 4], dtype=torch.int64) bitmask = allocate_token_bitmask(3, VOCAB_SIZE) # Wrong shape for retrieve_next_sibling with pytest.raises(RuntimeError): matcher.traverse_draft_tree( retrieve_next_token, torch.tensor([-1, -1], dtype=torch.int64), draft_tokens, bitmask ) # Wrong dtype for retrieve_next_sibling with pytest.raises(RuntimeError): matcher.traverse_draft_tree( retrieve_next_token, torch.tensor([-1, -1, -1], dtype=torch.int32), draft_tokens, bitmask, ) # Wrong rank for token_bitmask with pytest.raises(RuntimeError): matcher.traverse_draft_tree( retrieve_next_token, torch.tensor([-1, -1, -1], dtype=torch.int64), draft_tokens, torch.full((bitmask.shape[1],), -1, dtype=torch.int32), ) # Wrong batch size for token_bitmask with pytest.raises(RuntimeError): matcher.traverse_draft_tree( retrieve_next_token, torch.tensor([-1, -1, -1], dtype=torch.int64), draft_tokens, allocate_token_bitmask(2, VOCAB_SIZE), ) # Root should not have siblings with pytest.raises(RuntimeError): matcher.traverse_draft_tree( retrieve_next_token, torch.tensor([1, -1, -1], dtype=torch.int64), draft_tokens, bitmask ) # ── Timeout tests ──────────────────────────────────────────────────────────── def test_traverse_draft_tree_timeout_no_change(compiled_grammar): """Results should be identical whether time_threshold is omitted or set generously.""" for tree in [LINEAR_TREE, SIBLING_TREE]: # Baseline with timeout explicitly disabled _, bitmask_baseline = _run_traverse(compiled_grammar, tree, time_threshold=-1.0) # Omitting time_threshold entirely (exercises the Python default) result_default, bitmask_default = _run_traverse(compiled_grammar, tree) assert result_default is True assert torch.equal(bitmask_baseline, bitmask_default) # Large timeout should also produce identical results result_large, bitmask_large = _run_traverse(compiled_grammar, tree, time_threshold=100.0) assert result_large is True assert torch.equal(bitmask_baseline, bitmask_large) def test_traverse_draft_tree_timeout_triggers(): """A near-zero time_threshold should cause the traversal to time out. Timeout is only checked for non-root nodes, so the root bitmask is always computed. We build a deep linear chain so the timeout check is exercised. """ grammar = xgr.Grammar.from_ebnf('root ::= "a" root | "a"') tokenizer_info = xgr.TokenizerInfo(["a"], vocab_size=1, stop_token_ids=[]) compiler = xgr.GrammarCompiler(tokenizer_info) chain_grammar = compiler.compile_grammar(grammar) num_nodes = 10000 deep_chain = ( torch.tensor([i + 1 for i in range(num_nodes - 1)] + [-1], dtype=torch.int64), torch.full((num_nodes,), -1, dtype=torch.int64), torch.zeros(num_nodes, dtype=torch.int64), ) matcher = xgr.GrammarMatcher(chain_grammar) bitmask = allocate_token_bitmask(num_nodes, 1) result = matcher.traverse_draft_tree(*deep_chain, bitmask, time_threshold=1e-7) assert result is False, "Traversal should time out with near-zero threshold" # Root bitmask should still be filled since timeout is only checked for non-root nodes assert bitmask[0].any(), "Root bitmask should still be computed even when timed out" if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.2.3/tests/python/test_structural_tag_converter.py000066400000000000000000004416701521764210300245750ustar00rootroot00000000000000import sys import time from typing import Any, Dict, List, Optional, Tuple, Union import pytest from transformers import AutoTokenizer import xgrammar as xgr from xgrammar.structural_tag import StructuralTag from xgrammar.testing import _is_grammar_accept_string class Profiler: def __init__(self, tokenizer_id: str): tokenizer = AutoTokenizer.from_pretrained( tokenizer_id, use_fast=True, trust_remote_code=True ) self.tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) self.compiler = xgr.GrammarCompiler( self.tokenizer_info, max_threads=16, cache_enabled=False ) def profile_stag( self, structural_tag_format: Union[Dict[str, Any], StructuralTag], instance: str ): if isinstance(structural_tag_format, StructuralTag): structural_tag = structural_tag_format else: structural_tag = {"type": "structural_tag", "format": structural_tag_format} time_begin = time.monotonic_ns() compiled_grammar = self.compiler.compile_structural_tag(structural_tag) time_end = time.monotonic_ns() compiler_duration = time_end - time_begin print(f"Compiling structural tag {structural_tag_format}") print(f"Compile time: {compiler_duration / 1000 / 1000} ms") matcher = xgr.GrammarMatcher(compiled_grammar) token_bitmask = xgr.allocate_token_bitmask(1, self.tokenizer_info.vocab_size) print(f"Matching instance: {instance}") for char in instance: matcher.accept_string(char) time_begin = time.monotonic_ns() matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() duration = time_end - time_begin print(f"Time to generate mask: {duration / 1000} us, Character: '{char}'") profiler: Optional[Profiler] = None PROFILER_ON = True tokenizer_id = "meta-llama/Llama-3.1-8B-Instruct" @pytest.fixture(autouse=True, scope="module") def disable_profiler(request): global PROFILER_ON global profiler # Import shared token check from conftest (handles env vars + cached login) from conftest import _hf_token_available, _hf_token_explicitly_disabled if not _hf_token_available() or _hf_token_explicitly_disabled(request.config): PROFILER_ON = False else: profiler = Profiler(tokenizer_id) def check_stag_with_grammar(structural_tag_format: Dict[str, Any], expected_grammar_ebnf: str): structural_tag = {"type": "structural_tag", "format": structural_tag_format} stag_ebnf = xgr.Grammar.from_structural_tag(structural_tag) assert ( str(stag_ebnf) == expected_grammar_ebnf ), f"Expected:\n{expected_grammar_ebnf}\nGot:\n{str(stag_ebnf)}" def check_stag_with_instance( structural_tag_format: Union[Dict[str, Any], StructuralTag], instance: str, is_accepted: bool = True, debug_print: bool = False, ): if isinstance(structural_tag_format, StructuralTag): stag_grammar = xgr.Grammar.from_structural_tag(structural_tag_format) else: structural_tag = {"type": "structural_tag", "format": structural_tag_format} stag_grammar = xgr.Grammar.from_structural_tag(structural_tag) accepted = _is_grammar_accept_string(stag_grammar, instance, debug_print=debug_print) assert accepted == is_accepted if PROFILER_ON: profiler.profile_stag(structural_tag_format, instance) const_string_stag_grammar = [ ( {"type": "const_string", "value": "Hello!"}, r"""const_string ::= (("Hello!")) root ::= ((const_string)) """, ) ] const_string_instance_is_accepted = [ ("Hello!", True), ("Hello", False), ("Hello!!", False), ("HELLO!", False), ] def test_const_string_empty(): check_stag_with_instance({"type": "const_string", "value": ""}, "", True) check_stag_with_instance({"type": "const_string", "value": ""}, "x", False) @pytest.mark.parametrize("stag_format, expected_grammar", const_string_stag_grammar) @pytest.mark.parametrize("instance, is_accepted", const_string_instance_is_accepted) def test_const_string_format( stag_format: Dict[str, Any], expected_grammar: str, instance: str, is_accepted: bool ): check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, is_accepted, debug_print=True) json_schema_stag_grammar = [ ( { "type": "json_schema", "json_schema": {"type": "object", "properties": {"a": {"type": "string"}}}, }, r"""basic_escape ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) basic_string_sub ::= (("\"") | ([^\0-\x1f\"\\\r\n] basic_string_sub) | ("\\" basic_escape basic_string_sub)) (=([ \n\t]* [,}\]:])) basic_any ::= ((basic_number) | (basic_string) | (basic_boolean) | (basic_null) | (basic_array) | (basic_object)) basic_integer ::= (("0") | (basic_integer_1 [1-9] [0-9]*)) basic_number ::= ((basic_number_1 basic_number_7 basic_number_3 basic_number_6)) basic_string ::= (("\"" basic_string_sub)) basic_boolean ::= (("true") | ("false")) basic_null ::= (("null")) basic_array ::= (("[" [ \n\t]* basic_any basic_array_1 [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= (("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1 [ \n\t]* "}") | ("{" [ \n\t]* "}")) root_0 ::= (("{" [ \n\t]* "\"a\"" [ \n\t]* ":" [ \n\t]* basic_string [ \n\t]* "}") | ("{" [ \n\t]* "}")) basic_integer_1 ::= ("" | ("-")) basic_number_1 ::= ("" | ("-")) basic_number_2 ::= (([0-9] basic_number_2) | ([0-9])) basic_number_3 ::= ("" | ("." basic_number_2)) basic_number_4 ::= ("" | ([+\-])) basic_number_5 ::= (([0-9] basic_number_5) | ([0-9])) basic_number_6 ::= ("" | ([eE] basic_number_4 basic_number_5)) basic_array_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_any basic_array_1)) basic_object_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1)) basic_number_7 ::= (("0") | ([1-9] [0-9]*)) root ::= ((root_0)) """, ) ] json_schema_instance_is_accepted = [ ('{"a": "hello"}', True), ('{"a": 123}', False), ('{"b": "hello"}', False), ("invalid json", False), ] @pytest.mark.parametrize("stag_format, expected_grammar", json_schema_stag_grammar) @pytest.mark.parametrize("instance, is_accepted", json_schema_instance_is_accepted) def test_json_schema_format( stag_format: Dict[str, Any], expected_grammar: str, instance: str, is_accepted: bool ): check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, is_accepted) qwen_parameter_xml_stag_grammar = [ ( { "type": "qwen_xml_parameter", "json_schema": { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], }, }, r"""basic_escape ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) basic_string_sub ::= (("\"") | ([^\0-\x1f\"\\\r\n] basic_string_sub) | ("\\" basic_escape basic_string_sub)) (=([ \n\t]* [,}\]:])) basic_any ::= ((basic_number) | (basic_string) | (basic_boolean) | (basic_null) | (basic_array) | (basic_object)) basic_integer ::= (("0") | (basic_integer_1 [1-9] [0-9]*)) basic_number ::= ((basic_number_1 basic_number_7 basic_number_3 basic_number_6)) basic_string ::= (("\"" basic_string_sub)) basic_boolean ::= (("true") | ("false")) basic_null ::= (("null")) basic_array ::= (("[" [ \n\t]* basic_any basic_array_1 [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= (("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1 [ \n\t]* "}") | ("{" [ \n\t]* "}")) xml_string ::= TagDispatch( loop_after_dispatch=false, excludes=("") ) xml_any ::= ((xml_string) | (basic_array) | (basic_object)) xml_object ::= (([ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "" xml_object_1 [ \n\t]*) | ([ \n\t]*)) xml_variable_name ::= (([a-zA-Z_] [a-zA-Z0-9_]*)) root_prop_1 ::= (("0") | (root_prop_1_1 [1-9] [0-9]*)) root_part_0 ::= (([ \n\t]* "" [ \n\t]* root_prop_1 [ \n\t]* "")) root_0 ::= (([ \n\t]* "" [ \n\t]* xml_string [ \n\t]* "" root_part_0 [ \n\t]*)) basic_integer_1 ::= ("" | ("-")) basic_number_1 ::= ("" | ("-")) basic_number_2 ::= (([0-9] basic_number_2) | ([0-9])) basic_number_3 ::= ("" | ("." basic_number_2)) basic_number_4 ::= ("" | ([+\-])) basic_number_5 ::= (([0-9] basic_number_5) | ([0-9])) basic_number_6 ::= ("" | ([eE] basic_number_4 basic_number_5)) basic_array_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_any basic_array_1)) basic_object_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1)) xml_object_1 ::= ("" | ([ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "" xml_object_1)) root_prop_1_1 ::= ("" | ("-")) basic_number_7 ::= (("0") | ([1-9] [0-9]*)) root ::= ((root_0)) """, ) ] qwen_parameter_xml_instance_is_accepted = [ ("Bob\t100\n", True), ("Bob\t100\n", True), ("Bob100", True), ("\n\tBob100", True), ('"Bob<"100', True), ( """

Hello

100""", True, ), ] @pytest.mark.parametrize("stag_format, expected_grammar", qwen_parameter_xml_stag_grammar) @pytest.mark.parametrize("instance, is_accepted", qwen_parameter_xml_instance_is_accepted) def test_qwen_parameter_xml_format( stag_format: Dict[str, Any], expected_grammar: str, instance: str, is_accepted: bool ): check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, is_accepted) # JSONSchemaFormat with style="qwen_xml" (same behavior as qwen_xml_parameter) json_schema_style_qwen_xml_stag_grammar = [ ( { "type": "json_schema", "json_schema": { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], }, "style": "qwen_xml", }, qwen_parameter_xml_stag_grammar[0][1], # same expected grammar as qwen_xml_parameter ) ] @pytest.mark.parametrize("stag_format, expected_grammar", json_schema_style_qwen_xml_stag_grammar) @pytest.mark.parametrize("instance, is_accepted", qwen_parameter_xml_instance_is_accepted) def test_json_schema_style_qwen_xml_format( stag_format: Dict[str, Any], expected_grammar: str, instance: str, is_accepted: bool ): """Test JSONSchemaFormat with style='qwen_xml' produces same grammar and acceptance.""" check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, is_accepted) # JSONSchemaFormat with style="minimax_xml" (value) minimax_xml_instance_is_accepted = [ ('Bob\t100\n', True), ('Bob\t\n\t100\n', True), ('Bob100', True), ( """

Hello

100""", True, ), ] json_schema_style_minimax_xml_stag_grammar = [ ( { "type": "json_schema", "json_schema": { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], }, "style": "minimax_xml", }, r"""basic_escape ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) basic_string_sub ::= (("\"") | ([^\0-\x1f\"\\\r\n] basic_string_sub) | ("\\" basic_escape basic_string_sub)) (=([ \n\t]* [,}\]:])) basic_any ::= ((basic_number) | (basic_string) | (basic_boolean) | (basic_null) | (basic_array) | (basic_object)) basic_integer ::= (("0") | (basic_integer_1 [1-9] [0-9]*)) basic_number ::= ((basic_number_1 basic_number_7 basic_number_3 basic_number_6)) basic_string ::= (("\"" basic_string_sub)) basic_boolean ::= (("true") | ("false")) basic_null ::= (("null")) basic_array ::= (("[" [ \n\t]* basic_any basic_array_1 [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= (("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1 [ \n\t]* "}") | ("{" [ \n\t]* "}")) xml_string ::= TagDispatch( loop_after_dispatch=false, excludes=("") ) xml_any ::= ((xml_string) | (basic_array) | (basic_object)) xml_object ::= (([ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "" xml_object_1 [ \n\t]*) | ([ \n\t]*)) xml_variable_name ::= (([a-zA-Z_] [a-zA-Z0-9_]*)) root_prop_1 ::= (("0") | (root_prop_1_1 [1-9] [0-9]*)) root_part_0 ::= (([ \n\t]* "" [ \n\t]* root_prop_1 [ \n\t]* "")) root_0 ::= (([ \n\t]* "" [ \n\t]* xml_string [ \n\t]* "" root_part_0 [ \n\t]*)) basic_integer_1 ::= ("" | ("-")) basic_number_1 ::= ("" | ("-")) basic_number_2 ::= (([0-9] basic_number_2) | ([0-9])) basic_number_3 ::= ("" | ("." basic_number_2)) basic_number_4 ::= ("" | ([+\-])) basic_number_5 ::= (([0-9] basic_number_5) | ([0-9])) basic_number_6 ::= ("" | ([eE] basic_number_4 basic_number_5)) basic_array_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_any basic_array_1)) basic_object_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1)) xml_object_1 ::= ("" | ([ \n\t]* "" [ \n\t]* xml_any [ \n\t]* "" xml_object_1)) root_prop_1_1 ::= ("" | ("-")) basic_number_7 ::= (("0") | ([1-9] [0-9]*)) root ::= ((root_0)) """, ) ] @pytest.mark.parametrize( "stag_format, expected_grammar", json_schema_style_minimax_xml_stag_grammar ) @pytest.mark.parametrize("instance, is_accepted", minimax_xml_instance_is_accepted) def test_json_schema_style_minimax_xml_format( stag_format: Dict[str, Any], expected_grammar: str, instance: str, is_accepted: bool ): """Test JSONSchemaFormat with style='minimax_xml' (value).""" check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, is_accepted) # JSONSchemaFormat with style="deepseek_xml" (<｜DSML｜parameter name="key" string="true|false">value) deepseek_xml_instance_is_accepted = [ ( '<｜DSML｜parameter name="name" string="true">Bob<｜DSML｜parameter name="age" string="false">\t100\n', True, ), ( '<｜DSML｜parameter name="name" string="true">Bob\t\n<｜DSML｜parameter name="age" string="true">\t100\n', True, ), ( '<｜DSML｜parameter name="name" string="false">Bob<｜DSML｜parameter name="age" string="true">100', True, ), ( """<｜DSML｜parameter name="name" string="true">

Hello

<｜DSML｜parameter name="age" string="false">100""", True, ), ] json_schema_style_deepseek_xml_stag_grammar = [ ( { "type": "json_schema", "json_schema": { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], }, "style": "deepseek_xml", }, r"""basic_escape ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) basic_string_sub ::= (("\"") | ([^\0-\x1f\"\\\r\n] basic_string_sub) | ("\\" basic_escape basic_string_sub)) (=([ \n\t]* [,}\]:])) basic_any ::= ((basic_number) | (basic_string) | (basic_boolean) | (basic_null) | (basic_array) | (basic_object)) basic_integer ::= (("0") | (basic_integer_1 [1-9] [0-9]*)) basic_number ::= ((basic_number_1 basic_number_7 basic_number_3 basic_number_6)) basic_string ::= (("\"" basic_string_sub)) basic_boolean ::= (("true") | ("false")) basic_null ::= (("null")) basic_array ::= (("[" [ \n\t]* basic_any basic_array_1 [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= (("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1 [ \n\t]* "}") | ("{" [ \n\t]* "}")) xml_string ::= TagDispatch( loop_after_dispatch=false, excludes=("") ) xml_any ::= ((xml_string) | (basic_array) | (basic_object)) xml_object ::= (([ \n\t]* "<\uff5cDSML\uff5cparameter name=\"" xml_variable_name "\" string=\"" xml_object_2 "\">" [ \n\t]* xml_any [ \n\t]* "" xml_object_1 [ \n\t]*) | ([ \n\t]*)) xml_variable_name ::= (([a-zA-Z_] [a-zA-Z0-9_]*)) root_prop_1 ::= (("0") | (root_prop_1_1 [1-9] [0-9]*)) root_part_0 ::= (([ \n\t]* "<\uff5cDSML\uff5cparameter name=\"age\" string=\"" root_part_0_1 "\">" [ \n\t]* root_prop_1 [ \n\t]* "")) root_0 ::= (([ \n\t]* "<\uff5cDSML\uff5cparameter name=\"name\" string=\"" root_1 "\">" [ \n\t]* xml_string [ \n\t]* "" root_part_0 [ \n\t]*)) basic_integer_1 ::= ("" | ("-")) basic_number_1 ::= ("" | ("-")) basic_number_2 ::= (([0-9] basic_number_2) | ([0-9])) basic_number_3 ::= ("" | ("." basic_number_2)) basic_number_4 ::= ("" | ([+\-])) basic_number_5 ::= (([0-9] basic_number_5) | ([0-9])) basic_number_6 ::= ("" | ([eE] basic_number_4 basic_number_5)) basic_array_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_any basic_array_1)) basic_object_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1)) xml_object_1 ::= ("" | ([ \n\t]* "<\uff5cDSML\uff5cparameter name=\"" xml_variable_name "\" string=\"" xml_object_1_1 "\">" [ \n\t]* xml_any [ \n\t]* "" xml_object_1)) root_prop_1_1 ::= ("" | ("-")) basic_number_7 ::= (("0") | ([1-9] [0-9]*)) xml_object_2 ::= (("true") | ("false")) root_part_0_1 ::= (("true") | ("false")) root_1 ::= (("true") | ("false")) xml_object_1_1 ::= (("true") | ("false")) root ::= ((root_0)) """, ) ] @pytest.mark.parametrize( "stag_format, expected_grammar", json_schema_style_deepseek_xml_stag_grammar ) @pytest.mark.parametrize("instance, is_accepted", deepseek_xml_instance_is_accepted) def test_json_schema_style_deepseek_xml_format( stag_format: Dict[str, Any], expected_grammar: str, instance: str, is_accepted: bool ): """Test JSONSchemaFormat with style='deepseek_xml' (<｜DSML｜parameter name=\"key\" string=\"true|false\">value).""" check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, is_accepted) glm_xml_instance_is_accepted = [ ( "nameBobage100", True, ), ("nameBob", False), ("Bob100", False), ('Bob100', False), ] @pytest.mark.parametrize("instance, is_accepted", glm_xml_instance_is_accepted) def test_json_schema_style_glm_xml_format(instance: str, is_accepted: bool): """Test JSONSchemaFormat with style='glm_xml' (kv).""" stag_format = { "type": "json_schema", "json_schema": { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], }, "style": "glm_xml", } structural_tag = {"type": "structural_tag", "format": stag_format} stag_grammar = xgr.Grammar.from_structural_tag(structural_tag) grammar_str = str(stag_grammar) assert "" in grammar_str assert "" in grammar_str check_stag_with_instance(stag_format, instance, is_accepted) ebnf_grammar_stag_grammar = [ ( { "type": "grammar", "grammar": r"""root ::= "Hello!" number number ::= [0-9] | [0-9] number""", }, r"""root_0 ::= (("Hello!" number)) number ::= (([0-9]) | ([0-9] number)) root ::= ((root_0)) """, ) ] ebnf_grammar_instance_is_accepted = [ ("Hello!12345", True), ("Hello!0", True), ("Hello!", False), ("Hello!123a", False), ("Hi!123", False), ] @pytest.mark.parametrize("stag_format, expected_grammar", ebnf_grammar_stag_grammar) @pytest.mark.parametrize("instance, is_accepted", ebnf_grammar_instance_is_accepted) def test_ebnf_grammar_format( stag_format: Dict[str, Any], expected_grammar: str, instance: str, is_accepted: bool ): check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, is_accepted) regex_stag_grammar = [ ( {"type": "regex", "pattern": "Hello![0-9]+"}, r"""root_0 ::= (("H" "e" "l" "l" "o" "!" root_1)) root_1 ::= (([0-9] root_1) | ([0-9])) root ::= ((root_0)) """, ) ] regex_instance_is_accepted = [ ("Hello!12345", True), ("Hello!0", True), ("Hello!", False), ("Hello!123a", False), ("Hi!123", False), ] @pytest.mark.parametrize("stag_format, expected_grammar", regex_stag_grammar) @pytest.mark.parametrize("instance, is_accepted", regex_instance_is_accepted) def test_regex_format( stag_format: Dict[str, Any], expected_grammar: str, instance: str, is_accepted: bool ): check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, is_accepted) sequence_stag_grammar = [ ( { "type": "sequence", "elements": [ {"type": "const_string", "value": "Hello!"}, {"type": "json_schema", "json_schema": {"type": "number"}}, {"type": "grammar", "grammar": 'root ::= "" | [-+*/]'}, {"type": "regex", "pattern": "[simple]?"}, ], }, r"""const_string ::= (("Hello!")) basic_escape ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) basic_string_sub ::= (("\"") | ([^\0-\x1f\"\\\r\n] basic_string_sub) | ("\\" basic_escape basic_string_sub)) (=([ \n\t]* [,}\]:])) basic_any ::= ((basic_number) | (basic_string) | (basic_boolean) | (basic_null) | (basic_array) | (basic_object)) basic_integer ::= (("0") | (basic_integer_1 [1-9] [0-9]*)) basic_number ::= ((basic_number_1 basic_number_7 basic_number_3 basic_number_6)) basic_string ::= (("\"" basic_string_sub)) basic_boolean ::= (("true") | ("false")) basic_null ::= (("null")) basic_array ::= (("[" [ \n\t]* basic_any basic_array_1 [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= (("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1 [ \n\t]* "}") | ("{" [ \n\t]* "}")) root_0 ::= ((basic_number)) basic_integer_1 ::= ("" | ("-")) basic_number_1 ::= ("" | ("-")) basic_number_2 ::= (([0-9] basic_number_2) | ([0-9])) basic_number_3 ::= ("" | ("." basic_number_2)) basic_number_4 ::= ("" | ([+\-])) basic_number_5 ::= (([0-9] basic_number_5) | ([0-9])) basic_number_6 ::= ("" | ([eE] basic_number_4 basic_number_5)) basic_array_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_any basic_array_1)) basic_object_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1)) basic_number_7 ::= (("0") | ([1-9] [0-9]*)) root_1 ::= ("" | ([\-+*/])) root_2 ::= ((root_1_1)) root_1_1 ::= ("" | ([simple])) sequence ::= ((const_string root_0 root_1 root_2)) root ::= ((sequence)) """, ) ] sequence_instance_is_accepted = [ ("Hello!123", True), ("Hello!Hello!", False), ("Hello!", False), ("123Hello!", False), ("???", False), ("Hello!123+", True), ("Hello!123-", True), ("Hello!123!", False), ("Hello!123s", True), ("Hello!123+s", True), ("Hello!123q", False), ] @pytest.mark.parametrize("stag_format, expected_grammar", sequence_stag_grammar) @pytest.mark.parametrize("instance, is_accepted", sequence_instance_is_accepted) def test_sequence_format( stag_format: Dict[str, Any], expected_grammar: str, instance: str, is_accepted: bool ): check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, is_accepted) or_stag_grammar = [ ( { "type": "or", "elements": [ {"type": "const_string", "value": "Hello!"}, {"type": "json_schema", "json_schema": {"type": "number"}}, ], }, r"""const_string ::= (("Hello!")) basic_escape ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) basic_string_sub ::= (("\"") | ([^\0-\x1f\"\\\r\n] basic_string_sub) | ("\\" basic_escape basic_string_sub)) (=([ \n\t]* [,}\]:])) basic_any ::= ((basic_number) | (basic_string) | (basic_boolean) | (basic_null) | (basic_array) | (basic_object)) basic_integer ::= (("0") | (basic_integer_1 [1-9] [0-9]*)) basic_number ::= ((basic_number_1 basic_number_7 basic_number_3 basic_number_6)) basic_string ::= (("\"" basic_string_sub)) basic_boolean ::= (("true") | ("false")) basic_null ::= (("null")) basic_array ::= (("[" [ \n\t]* basic_any basic_array_1 [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= (("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1 [ \n\t]* "}") | ("{" [ \n\t]* "}")) root_0 ::= ((basic_number)) basic_integer_1 ::= ("" | ("-")) basic_number_1 ::= ("" | ("-")) basic_number_2 ::= (([0-9] basic_number_2) | ([0-9])) basic_number_3 ::= ("" | ("." basic_number_2)) basic_number_4 ::= ("" | ([+\-])) basic_number_5 ::= (([0-9] basic_number_5) | ([0-9])) basic_number_6 ::= ("" | ([eE] basic_number_4 basic_number_5)) basic_array_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_any basic_array_1)) basic_object_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1)) basic_number_7 ::= (("0") | ([1-9] [0-9]*)) or ::= ((const_string) | (root_0)) root ::= ((or)) """, ) ] or_instance_is_accepted = [ ("Hello!", True), ("123", True), ("Hello!Hello!", False), ("123Hello!", False), ("???", False), ] @pytest.mark.parametrize("stag_format, expected_grammar", or_stag_grammar) @pytest.mark.parametrize("instance, is_accepted", or_instance_is_accepted) def test_or_format( stag_format: Dict[str, Any], expected_grammar: str, instance: str, is_accepted: bool ): check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, is_accepted) tag_stag_grammar = [ ( { "type": "tag", "begin": "BEG", "content": {"type": "json_schema", "json_schema": {"type": "number"}}, "end": "END", }, r"""basic_escape ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) basic_string_sub ::= (("\"") | ([^\0-\x1f\"\\\r\n] basic_string_sub) | ("\\" basic_escape basic_string_sub)) (=([ \n\t]* [,}\]:])) basic_any ::= ((basic_number) | (basic_string) | (basic_boolean) | (basic_null) | (basic_array) | (basic_object)) basic_integer ::= (("0") | (basic_integer_1 [1-9] [0-9]*)) basic_number ::= ((basic_number_1 basic_number_7 basic_number_3 basic_number_6)) basic_string ::= (("\"" basic_string_sub)) basic_boolean ::= (("true") | ("false")) basic_null ::= (("null")) basic_array ::= (("[" [ \n\t]* basic_any basic_array_1 [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= (("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1 [ \n\t]* "}") | ("{" [ \n\t]* "}")) root_0 ::= ((basic_number)) basic_integer_1 ::= ("" | ("-")) basic_number_1 ::= ("" | ("-")) basic_number_2 ::= (([0-9] basic_number_2) | ([0-9])) basic_number_3 ::= ("" | ("." basic_number_2)) basic_number_4 ::= ("" | ([+\-])) basic_number_5 ::= (([0-9] basic_number_5) | ([0-9])) basic_number_6 ::= ("" | ([eE] basic_number_4 basic_number_5)) basic_array_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_any basic_array_1)) basic_object_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1)) basic_number_7 ::= (("0") | ([1-9] [0-9]*)) tag ::= (("BEG" root_0 "END")) root ::= ((tag)) """, ), ( { "type": "tag", "begin": "BEG", "content": {"type": "grammar", "grammar": "root ::= [+\\-]?[1-9][0-9]*"}, "end": "END", }, r"""root_0 ::= ((root_1 [1-9] [0-9]*)) root_1 ::= ("" | ([+\-])) tag ::= (("BEG" root_0 "END")) root ::= ((tag)) """, ), ( { "type": "tag", "begin": "BEG", "content": {"type": "regex", "pattern": "[+\\-]?[1-9][0-9]*"}, "end": "END", }, r"""root_0 ::= ((root_1 [1-9] [0-9]*)) root_1 ::= ("" | ([+\-])) tag ::= (("BEG" root_0 "END")) root ::= ((tag)) """, ), ] tag_instance_is_accepted = [ ("BEG12345END", True), ("BEG123456END", True), ("BEG1234567END", True), ("BEG???END", False), ("BEG12345ENDEND", False), ] @pytest.mark.parametrize("stag_format, expected_grammar", tag_stag_grammar) @pytest.mark.parametrize("instance, is_accepted", tag_instance_is_accepted) def test_tag_format( stag_format: Dict[str, Any], expected_grammar: str, instance: str, is_accepted: bool ): check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, is_accepted) any_text_stag_grammar = [ ( {"type": "tag", "begin": "BEG", "content": {"type": "any_text"}, "end": "END"}, r"""any_text ::= TagDispatch( loop_after_dispatch=false, excludes=("END") ) tag ::= (("BEG" any_text "END")) root ::= ((tag)) """, ) ] any_text_instance_is_accepted = [ ("BEGHello!END", True), ("BEGENENNDENEND", True), ("BEGENENDEN", False), ("BEGBEGENDEND", False), ] @pytest.mark.parametrize("stag_format, expected_grammar", any_text_stag_grammar) @pytest.mark.parametrize("instance, is_accepted", any_text_instance_is_accepted) def test_any_text_format( stag_format: Dict[str, Any], expected_grammar: str, instance: str, is_accepted: bool ): check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, is_accepted) any_text_only_stag_grammar = [ ( {"type": "any_text"}, r"""any_text ::= (([\0-\U0010ffff]*)) root ::= ((any_text)) """, ) ] any_text_only_instance_is_accepted = [("ABCDEF", True), ("123456", True), ("", True)] @pytest.mark.parametrize("stag_format, expected_grammar", any_text_only_stag_grammar) @pytest.mark.parametrize("instance, is_accepted", any_text_only_instance_is_accepted) def test_any_text_only_format( stag_format: Dict[str, Any], expected_grammar: str, instance: str, is_accepted: bool ): check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, is_accepted) test_no_end_anytext_format_with_excludes_instance_is_accepted = [ ("hello world", True), ("hello world", True), ("", True), ] @pytest.mark.parametrize( "instance, is_accepted", test_no_end_anytext_format_with_excludes_instance_is_accepted ) def test_no_end_anytext_format_with_excludes(instance: str, is_accepted: bool): stag_format = { "type": "triggered_tags", "triggers": [""], "tags": [ {"begin": "", "content": {"type": "any_text", "excludes": [""]}, "end": ""} ], "at_least_one": True, } expected_grammar = r"""any_text ::= TagDispatch( loop_after_dispatch=false, excludes=("") ) triggered_tags_group ::= (("" any_text)) triggered_tags_first ::= (("" any_text)) triggered_tags_sub ::= TagDispatch( ("", triggered_tags_group), loop_after_dispatch=true, excludes=() ) triggered_tags ::= ((triggered_tags_first triggered_tags_sub)) root ::= ((triggered_tags)) """ check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, is_accepted) def _get_triggered_tag_format(at_least_one: bool, stop_after_first: bool): return { "type": "triggered_tags", "triggers": ["A"], "tags": [ {"begin": "A1", "content": {"type": "const_string", "value": "L1"}, "end": "A"}, {"begin": "A2", "content": {"type": "const_string", "value": "L2"}, "end": "A"}, ], "at_least_one": at_least_one, "stop_after_first": stop_after_first, } triggered_tag_stag_grammar = [ ( 0, _get_triggered_tag_format(at_least_one=False, stop_after_first=False), r"""const_string ::= (("L1")) const_string_1 ::= (("L2")) triggered_tags_group ::= (("1" const_string "A") | ("2" const_string_1 "A")) triggered_tags ::= TagDispatch( ("A", triggered_tags_group), loop_after_dispatch=true, excludes=() ) root ::= ((triggered_tags)) """, ), ( 1, _get_triggered_tag_format(at_least_one=True, stop_after_first=False), r"""const_string ::= (("L1")) const_string_1 ::= (("L2")) triggered_tags_group ::= (("1" const_string "A") | ("2" const_string_1 "A")) triggered_tags_first ::= (("A1" const_string "A") | ("A2" const_string_1 "A")) triggered_tags_sub ::= TagDispatch( ("A", triggered_tags_group), loop_after_dispatch=true, excludes=() ) triggered_tags ::= ((triggered_tags_first triggered_tags_sub)) root ::= ((triggered_tags)) """, ), ( 2, _get_triggered_tag_format(at_least_one=False, stop_after_first=True), r"""const_string ::= (("L1")) const_string_1 ::= (("L2")) triggered_tags_group ::= (("1" const_string "A") | ("2" const_string_1 "A")) triggered_tags ::= TagDispatch( ("A", triggered_tags_group), loop_after_dispatch=false, excludes=() ) root ::= ((triggered_tags)) """, ), ( 3, _get_triggered_tag_format(at_least_one=True, stop_after_first=True), r"""const_string ::= (("L1")) const_string_1 ::= (("L2")) triggered_tags ::= (("A1" const_string "A") | ("A2" const_string_1 "A")) root ::= ((triggered_tags)) """, ), ] triggered_tag_instance_accepted_results = [ ("textA1L1AtextA2L2AText", [True, False, False, False]), ("textA1L1AtextA2L2A", [True, False, False, False]), ("A1L1Atext", [True, True, False, False]), ("A1L1AtextA2L2A", [True, True, False, False]), ("A1L1A", [True, True, True, True]), ("text", [True, False, True, False]), ("", [True, False, True, False]), ("AA", [False, False, False, False]), ("A1L2A", [False, False, False, False]), ("A1L1A2L2A", [False, False, False, False]), ] @pytest.mark.parametrize("stag_id, stag_format, expected_grammar", triggered_tag_stag_grammar) @pytest.mark.parametrize("instance, accepted_results", triggered_tag_instance_accepted_results) def test_triggered_tag_format( stag_id: int, stag_format: Dict[str, Any], expected_grammar: str, instance: str, accepted_results: List[bool], ): check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, accepted_results[stag_id]) test_triggered_tags_corner_case_data = [ ( { "type": "triggered_tags", "triggers": [""], "tags": [ { "begin": "", "content": {"type": "const_string", "value": "[TEXT]"}, "end": "", } ], }, r"""const_string ::= (("[TEXT]")) triggered_tags_group ::= (("" const_string "")) triggered_tags ::= TagDispatch( ("", triggered_tags_group), loop_after_dispatch=true, excludes=() ) root ::= ((triggered_tags)) """, [("[TEXT][TEXT][TEXT][TEXT]", True)], ) ] @pytest.mark.parametrize( "stag_format, expected_grammar, instance_is_accepted_tuples", test_triggered_tags_corner_case_data, ) def test_triggered_tags_corner_case( stag_format: Dict[str, Any], expected_grammar: str, instance_is_accepted_tuples: List[Tuple[str, bool]], ): check_stag_with_grammar(stag_format, expected_grammar) for instance, is_accepted in instance_is_accepted_tuples: check_stag_with_instance(stag_format, instance, is_accepted) triggered_tag_format = { "type": "triggered_tags", "triggers": ["A"], "tags": [ {"begin": "A1", "content": {"type": "const_string", "value": "L1"}, "end": "A"}, {"begin": "A2", "content": {"type": "const_string", "value": "L2"}, "end": "A"}, ], } def _get_triggered_tag_with_outside_tag(at_least_one: bool, stop_after_first: bool): return { "type": "tag", "begin": "begin", "content": { "type": "triggered_tags", "triggers": ["A"], "tags": [ {"begin": "A1", "content": {"type": "const_string", "value": "L1"}, "end": "A"}, {"begin": "A2", "content": {"type": "const_string", "value": "L2"}, "end": "A"}, ], "at_least_one": at_least_one, "stop_after_first": stop_after_first, }, "end": "end", } triggered_tag_with_outside_tag_stag_grammar = [ ( 0, _get_triggered_tag_with_outside_tag(at_least_one=False, stop_after_first=False), r"""const_string ::= (("L1")) const_string_1 ::= (("L2")) triggered_tags_group ::= (("1" const_string "A") | ("2" const_string_1 "A")) triggered_tags ::= TagDispatch( ("A", triggered_tags_group), loop_after_dispatch=true, excludes=("end") ) tag ::= (("begin" triggered_tags "end")) root ::= ((tag)) """, ), ( 1, _get_triggered_tag_with_outside_tag(at_least_one=True, stop_after_first=False), r"""const_string ::= (("L1")) const_string_1 ::= (("L2")) triggered_tags_group ::= (("1" const_string "A") | ("2" const_string_1 "A")) triggered_tags_first ::= (("A1" const_string "A") | ("A2" const_string_1 "A")) triggered_tags_sub ::= TagDispatch( ("A", triggered_tags_group), loop_after_dispatch=true, excludes=("end") ) triggered_tags ::= ((triggered_tags_first triggered_tags_sub)) tag ::= (("begin" triggered_tags "end")) root ::= ((tag)) """, ), ( 2, _get_triggered_tag_with_outside_tag(at_least_one=False, stop_after_first=True), r"""const_string ::= (("L1")) const_string_1 ::= (("L2")) triggered_tags_group ::= (("1" const_string "A") | ("2" const_string_1 "A")) triggered_tags ::= TagDispatch( ("A", triggered_tags_group), loop_after_dispatch=false, excludes=("end") ) tag ::= (("begin" triggered_tags "end")) root ::= ((tag)) """, ), ( 3, _get_triggered_tag_with_outside_tag(at_least_one=True, stop_after_first=True), r"""const_string ::= (("L1")) const_string_1 ::= (("L2")) triggered_tags ::= (("A1" const_string "A") | ("A2" const_string_1 "A")) tag ::= (("begin" triggered_tags "end")) root ::= ((tag)) """, ), ] triggered_tag_with_outside_tag_instance_accepted_results = [ ("beginabcA1L1Atextend", [True, False, False, False]), ("beginA1L1AtextA2L2Aend", [True, True, False, False]), ("beginA1L1Aend", [True, True, True, True]), ("beginend", [True, False, True, False]), ("beginA1L1Aendabc", [False, False, False, False]), ("beginA1L2end", [False, False, False, False]), ] @pytest.mark.parametrize( "stag_id, stag_format, expected_grammar", triggered_tag_with_outside_tag_stag_grammar ) @pytest.mark.parametrize( "instance, accepted_results", triggered_tag_with_outside_tag_instance_accepted_results ) def test_triggered_tag_with_outside_tag( stag_id: int, stag_format: Dict[str, Any], expected_grammar: str, instance: str, accepted_results: List[bool], ): check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, accepted_results[stag_id]) def _get_tags_with_separator_format(at_least_one: bool, stop_after_first: bool): return { "type": "tags_with_separator", "tags": [ {"begin": "A1", "content": {"type": "const_string", "value": "L1"}, "end": "A"}, {"begin": "A2", "content": {"type": "const_string", "value": "L2"}, "end": "A"}, ], "separator": "AA", "at_least_one": at_least_one, "stop_after_first": stop_after_first, } tags_with_separator_stag_grammar = [ ( 0, _get_tags_with_separator_format(at_least_one=False, stop_after_first=False), r"""const_string ::= (("L1")) tag ::= (("A1" const_string "A")) const_string_1 ::= (("L2")) tag_1 ::= (("A2" const_string_1 "A")) tags_with_separator_tags ::= ((tag) | (tag_1)) tags_with_separator_sub ::= ("" | ("AA" tags_with_separator_tags tags_with_separator_sub)) tags_with_separator ::= ("" | (tags_with_separator_tags tags_with_separator_sub)) root ::= ((tags_with_separator)) """, ), ( 1, _get_tags_with_separator_format(at_least_one=True, stop_after_first=False), r"""const_string ::= (("L1")) tag ::= (("A1" const_string "A")) const_string_1 ::= (("L2")) tag_1 ::= (("A2" const_string_1 "A")) tags_with_separator_tags ::= ((tag) | (tag_1)) tags_with_separator_sub ::= ("" | ("AA" tags_with_separator_tags tags_with_separator_sub)) tags_with_separator ::= ((tags_with_separator_tags tags_with_separator_sub)) root ::= ((tags_with_separator)) """, ), ( 2, _get_tags_with_separator_format(at_least_one=False, stop_after_first=True), r"""const_string ::= (("L1")) tag ::= (("A1" const_string "A")) const_string_1 ::= (("L2")) tag_1 ::= (("A2" const_string_1 "A")) tags_with_separator_tags ::= ((tag) | (tag_1)) tags_with_separator ::= ("" | (tags_with_separator_tags)) root ::= ((tags_with_separator)) """, ), ( 3, _get_tags_with_separator_format(at_least_one=True, stop_after_first=True), r"""const_string ::= (("L1")) tag ::= (("A1" const_string "A")) const_string_1 ::= (("L2")) tag_1 ::= (("A2" const_string_1 "A")) tags_with_separator_tags ::= ((tag) | (tag_1)) tags_with_separator ::= ((tags_with_separator_tags)) root ::= ((tags_with_separator)) """, ), ] tags_with_separator_instance_accepted_results = [ ("", [True, False, True, False]), ("A1L1A", [True, True, True, True]), ("A1L1AAAA2L2A", [True, True, False, False]), ("A1L1AA2L2A", [False, False, False, False]), ] @pytest.mark.parametrize("stag_id, stag_format, expected_grammar", tags_with_separator_stag_grammar) @pytest.mark.parametrize( "instance, accepted_results", tags_with_separator_instance_accepted_results ) def test_tags_with_separator_format( stag_id: int, stag_format: Dict[str, Any], expected_grammar: str, instance: str, accepted_results: List[bool], ): check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, accepted_results[stag_id]) def _get_tags_with_separator_format_with_outside_tag(at_least_one: bool, stop_after_first: bool): return { "type": "tag", "begin": "begin", "content": { "type": "tags_with_separator", "tags": [ {"begin": "A1", "content": {"type": "const_string", "value": "L1"}, "end": "A"}, {"begin": "A2", "content": {"type": "const_string", "value": "L2"}, "end": "A"}, ], "separator": "AA", "at_least_one": at_least_one, "stop_after_first": stop_after_first, }, "end": "end", } tags_with_separator_with_outside_tag_stag_grammar = [ ( 0, _get_tags_with_separator_format_with_outside_tag( at_least_one=False, stop_after_first=False ), r"""const_string ::= (("L1")) tag ::= (("A1" const_string "A")) const_string_1 ::= (("L2")) tag_1 ::= (("A2" const_string_1 "A")) tags_with_separator_tags ::= ((tag) | (tag_1)) tags_with_separator_sub ::= ("" | ("AA" tags_with_separator_tags tags_with_separator_sub)) tags_with_separator ::= ("" | (tags_with_separator_tags tags_with_separator_sub)) tag_2 ::= (("begin" tags_with_separator "end")) root ::= ((tag_2)) """, ), ( 1, _get_tags_with_separator_format_with_outside_tag(at_least_one=True, stop_after_first=False), r"""const_string ::= (("L1")) tag ::= (("A1" const_string "A")) const_string_1 ::= (("L2")) tag_1 ::= (("A2" const_string_1 "A")) tags_with_separator_tags ::= ((tag) | (tag_1)) tags_with_separator_sub ::= ("" | ("AA" tags_with_separator_tags tags_with_separator_sub)) tags_with_separator ::= ((tags_with_separator_tags tags_with_separator_sub)) tag_2 ::= (("begin" tags_with_separator "end")) root ::= ((tag_2)) """, ), ( 2, _get_tags_with_separator_format_with_outside_tag(at_least_one=False, stop_after_first=True), r"""const_string ::= (("L1")) tag ::= (("A1" const_string "A")) const_string_1 ::= (("L2")) tag_1 ::= (("A2" const_string_1 "A")) tags_with_separator_tags ::= ((tag) | (tag_1)) tags_with_separator ::= ("" | (tags_with_separator_tags)) tag_2 ::= (("begin" tags_with_separator "end")) root ::= ((tag_2)) """, ), ( 3, _get_tags_with_separator_format_with_outside_tag(at_least_one=True, stop_after_first=True), r"""const_string ::= (("L1")) tag ::= (("A1" const_string "A")) const_string_1 ::= (("L2")) tag_1 ::= (("A2" const_string_1 "A")) tags_with_separator_tags ::= ((tag) | (tag_1)) tags_with_separator ::= ((tags_with_separator_tags)) tag_2 ::= (("begin" tags_with_separator "end")) root ::= ((tag_2)) """, ), ] tags_with_separator_with_outside_tag_instance_accepted_results = [ ("beginend", [True, False, True, False]), ("beginA1L1Aend", [True, True, True, True]), ("beginA1L1AAAA2L2Aend", [True, True, False, False]), ("beginA1L1A", [False, False, False, False]), ("beginA1L1AA2L2Aend", [False, False, False, False]), ] @pytest.mark.parametrize( "stag_id, stag_format, expected_grammar", tags_with_separator_with_outside_tag_stag_grammar ) @pytest.mark.parametrize( "instance, accepted_results", tags_with_separator_with_outside_tag_instance_accepted_results ) def test_tags_with_separator_format_with_outside_tag( stag_id: int, stag_format: Dict[str, Any], expected_grammar: str, instance: str, accepted_results: List[bool], ): check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, accepted_results[stag_id]) # Test for empty separator in tags_with_separator def _get_tags_with_empty_separator_format(at_least_one: bool, stop_after_first: bool): return { "type": "tags_with_separator", "tags": [ {"begin": "", "content": {"type": "const_string", "value": "X"}, "end": ""}, {"begin": "", "content": {"type": "const_string", "value": "Y"}, "end": ""}, ], "separator": "", "at_least_one": at_least_one, "stop_after_first": stop_after_first, } tags_with_empty_separator_stag_grammar = [ ( 0, _get_tags_with_empty_separator_format(at_least_one=False, stop_after_first=False), r"""const_string ::= (("X")) tag ::= (("" const_string "")) const_string_1 ::= (("Y")) tag_1 ::= (("" const_string_1 "")) tags_with_separator_tags ::= ((tag) | (tag_1)) tags_with_separator_sub ::= ("" | (tags_with_separator_tags tags_with_separator_sub)) tags_with_separator ::= ("" | (tags_with_separator_tags tags_with_separator_sub)) root ::= ((tags_with_separator)) """, ), ( 1, _get_tags_with_empty_separator_format(at_least_one=True, stop_after_first=False), r"""const_string ::= (("X")) tag ::= (("" const_string "")) const_string_1 ::= (("Y")) tag_1 ::= (("" const_string_1 "")) tags_with_separator_tags ::= ((tag) | (tag_1)) tags_with_separator_sub ::= ("" | (tags_with_separator_tags tags_with_separator_sub)) tags_with_separator ::= ((tags_with_separator_tags tags_with_separator_sub)) root ::= ((tags_with_separator)) """, ), ( 2, _get_tags_with_empty_separator_format(at_least_one=False, stop_after_first=True), r"""const_string ::= (("X")) tag ::= (("" const_string "")) const_string_1 ::= (("Y")) tag_1 ::= (("" const_string_1 "")) tags_with_separator_tags ::= ((tag) | (tag_1)) tags_with_separator ::= ("" | (tags_with_separator_tags)) root ::= ((tags_with_separator)) """, ), ( 3, _get_tags_with_empty_separator_format(at_least_one=True, stop_after_first=True), r"""const_string ::= (("X")) tag ::= (("" const_string "")) const_string_1 ::= (("Y")) tag_1 ::= (("" const_string_1 "")) tags_with_separator_tags ::= ((tag) | (tag_1)) tags_with_separator ::= ((tags_with_separator_tags)) root ::= ((tags_with_separator)) """, ), ] tags_with_empty_separator_instance_accepted_results = [ ("", [True, False, True, False]), ("X", [True, True, True, True]), ("XY", [True, True, False, False]), ("YXY", [True, True, False, False]), ("XXX", [True, True, False, False]), # Invalid cases ("X,Y", [False, False, False, False]), # Has separator when none expected ("Z", [False, False, False, False]), # Unknown tag ] @pytest.mark.parametrize( "stag_id, stag_format, expected_grammar", tags_with_empty_separator_stag_grammar ) @pytest.mark.parametrize( "instance, accepted_results", tags_with_empty_separator_instance_accepted_results ) def test_tags_with_empty_separator_format( stag_id: int, stag_format: Dict[str, Any], expected_grammar: str, instance: str, accepted_results: List[bool], ): check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, accepted_results[stag_id]) # ---------- OptionalFormat (0 or 1 occurrence) ---------- optional_stag_grammar = [ ( 0, {"type": "optional", "content": {"type": "const_string", "value": "x"}}, r"""const_string ::= (("x")) optional ::= ("" | (const_string)) root ::= ((optional)) """, ), ( 1, { "type": "optional", "content": { "type": "sequence", "elements": [ {"type": "const_string", "value": "a"}, {"type": "const_string", "value": "b"}, ], }, }, r"""const_string ::= (("a")) const_string_1 ::= (("b")) sequence ::= ((const_string const_string_1)) optional ::= ("" | (sequence)) root ::= ((optional)) """, ), ( 2, { "type": "optional", "content": { "type": "or", "elements": [ {"type": "const_string", "value": "A"}, {"type": "const_string", "value": "B"}, ], }, }, r"""const_string ::= (("A")) const_string_1 ::= (("B")) or ::= ((const_string) | (const_string_1)) optional ::= ("" | (or)) root ::= ((optional)) """, ), ( 3, { "type": "optional", "content": { "type": "tag", "begin": "BEG", "content": {"type": "json_schema", "json_schema": {"type": "number"}}, "end": "END", }, }, r"""basic_escape ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) basic_string_sub ::= (("\"") | ([^\0-\x1f\"\\\r\n] basic_string_sub) | ("\\" basic_escape basic_string_sub)) (=([ \n\t]* [,}\]:])) basic_any ::= ((basic_number) | (basic_string) | (basic_boolean) | (basic_null) | (basic_array) | (basic_object)) basic_integer ::= (("0") | (basic_integer_1 [1-9] [0-9]*)) basic_number ::= ((basic_number_1 basic_number_7 basic_number_3 basic_number_6)) basic_string ::= (("\"" basic_string_sub)) basic_boolean ::= (("true") | ("false")) basic_null ::= (("null")) basic_array ::= (("[" [ \n\t]* basic_any basic_array_1 [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= (("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1 [ \n\t]* "}") | ("{" [ \n\t]* "}")) root_0 ::= ((basic_number)) basic_integer_1 ::= ("" | ("-")) basic_number_1 ::= ("" | ("-")) basic_number_2 ::= (([0-9] basic_number_2) | ([0-9])) basic_number_3 ::= ("" | ("." basic_number_2)) basic_number_4 ::= ("" | ([+\-])) basic_number_5 ::= (([0-9] basic_number_5) | ([0-9])) basic_number_6 ::= ("" | ([eE] basic_number_4 basic_number_5)) basic_array_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_any basic_array_1)) basic_object_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1)) basic_number_7 ::= (("0") | ([1-9] [0-9]*)) tag ::= (("BEG" root_0 "END")) optional ::= ("" | (tag)) root ::= ((optional)) """, ), ( 4, {"type": "optional", "content": {"type": "json_schema", "json_schema": {"type": "number"}}}, r"""basic_escape ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) basic_string_sub ::= (("\"") | ([^\0-\x1f\"\\\r\n] basic_string_sub) | ("\\" basic_escape basic_string_sub)) (=([ \n\t]* [,}\]:])) basic_any ::= ((basic_number) | (basic_string) | (basic_boolean) | (basic_null) | (basic_array) | (basic_object)) basic_integer ::= (("0") | (basic_integer_1 [1-9] [0-9]*)) basic_number ::= ((basic_number_1 basic_number_7 basic_number_3 basic_number_6)) basic_string ::= (("\"" basic_string_sub)) basic_boolean ::= (("true") | ("false")) basic_null ::= (("null")) basic_array ::= (("[" [ \n\t]* basic_any basic_array_1 [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= (("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1 [ \n\t]* "}") | ("{" [ \n\t]* "}")) root_0 ::= ((basic_number)) basic_integer_1 ::= ("" | ("-")) basic_number_1 ::= ("" | ("-")) basic_number_2 ::= (([0-9] basic_number_2) | ([0-9])) basic_number_3 ::= ("" | ("." basic_number_2)) basic_number_4 ::= ("" | ([+\-])) basic_number_5 ::= (([0-9] basic_number_5) | ([0-9])) basic_number_6 ::= ("" | ([eE] basic_number_4 basic_number_5)) basic_array_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_any basic_array_1)) basic_object_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1)) basic_number_7 ::= (("0") | ([1-9] [0-9]*)) optional ::= ("" | (root_0)) root ::= ((optional)) """, ), ] optional_instance_accepted_results = [ ("", [True, True, True, True, True]), ("x", [True, False, False, False, False]), ("ab", [False, True, False, False, False]), ("A", [False, False, True, False, False]), ("B", [False, False, True, False, False]), ("BEG42END", [False, False, False, True, False]), ("42", [False, False, False, False, True]), ("-3.14", [False, False, False, False, True]), ("xx", [False, False, False, False, False]), ("abab", [False, False, False, False, False]), ("AB", [False, False, False, False, False]), ("BEG1ENDBEG2END", [False, False, False, False, False]), ("invalid", [False, False, False, False, False]), ] @pytest.mark.parametrize("stag_id, stag_format, expected_grammar", optional_stag_grammar) @pytest.mark.parametrize("instance, accepted_results", optional_instance_accepted_results) def test_optional_format( stag_id: int, stag_format: Dict[str, Any], expected_grammar: str, instance: str, accepted_results: List[bool], ): check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, accepted_results[stag_id]) # ---------- PlusFormat (1 or more occurrences) ---------- plus_stag_grammar = [ ( 0, {"type": "plus", "content": {"type": "const_string", "value": "x"}}, r"""const_string ::= (("x")) plus_star ::= ("" | (const_string plus_star)) plus ::= ((const_string plus_star)) root ::= ((plus)) """, ), ( 1, { "type": "plus", "content": { "type": "sequence", "elements": [ {"type": "const_string", "value": "a"}, {"type": "const_string", "value": "b"}, ], }, }, r"""const_string ::= (("a")) const_string_1 ::= (("b")) sequence ::= ((const_string const_string_1)) plus_star ::= ("" | (sequence plus_star)) plus ::= ((sequence plus_star)) root ::= ((plus)) """, ), ( 2, { "type": "plus", "content": { "type": "or", "elements": [ {"type": "const_string", "value": "A"}, {"type": "const_string", "value": "B"}, ], }, }, r"""const_string ::= (("A")) const_string_1 ::= (("B")) or ::= ((const_string) | (const_string_1)) plus_star ::= ("" | (or plus_star)) plus ::= ((or plus_star)) root ::= ((plus)) """, ), ( 3, { "type": "plus", "content": { "type": "tag", "begin": "BEG", "content": {"type": "json_schema", "json_schema": {"type": "number"}}, "end": "END", }, }, r"""basic_escape ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) basic_string_sub ::= (("\"") | ([^\0-\x1f\"\\\r\n] basic_string_sub) | ("\\" basic_escape basic_string_sub)) (=([ \n\t]* [,}\]:])) basic_any ::= ((basic_number) | (basic_string) | (basic_boolean) | (basic_null) | (basic_array) | (basic_object)) basic_integer ::= (("0") | (basic_integer_1 [1-9] [0-9]*)) basic_number ::= ((basic_number_1 basic_number_7 basic_number_3 basic_number_6)) basic_string ::= (("\"" basic_string_sub)) basic_boolean ::= (("true") | ("false")) basic_null ::= (("null")) basic_array ::= (("[" [ \n\t]* basic_any basic_array_1 [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= (("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1 [ \n\t]* "}") | ("{" [ \n\t]* "}")) root_0 ::= ((basic_number)) basic_integer_1 ::= ("" | ("-")) basic_number_1 ::= ("" | ("-")) basic_number_2 ::= (([0-9] basic_number_2) | ([0-9])) basic_number_3 ::= ("" | ("." basic_number_2)) basic_number_4 ::= ("" | ([+\-])) basic_number_5 ::= (([0-9] basic_number_5) | ([0-9])) basic_number_6 ::= ("" | ([eE] basic_number_4 basic_number_5)) basic_array_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_any basic_array_1)) basic_object_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1)) basic_number_7 ::= (("0") | ([1-9] [0-9]*)) tag ::= (("BEG" root_0 "END")) plus_star ::= ("" | (tag plus_star)) plus ::= ((tag plus_star)) root ::= ((plus)) """, ), ( 4, { "type": "plus", "content": {"type": "optional", "content": {"type": "const_string", "value": "y"}}, }, r"""const_string ::= (("y")) optional ::= ("" | (const_string)) plus_star ::= ("" | (optional plus_star)) plus ::= ((optional plus_star)) root ::= ((plus)) """, ), ] plus_instance_accepted_results = [ ("", [False, False, False, False, True]), ("x", [True, False, False, False, False]), ("xx", [True, False, False, False, False]), ("xxx", [True, False, False, False, False]), ("ab", [False, True, False, False, False]), ("abab", [False, True, False, False, False]), ("ababab", [False, True, False, False, False]), ("A", [False, False, True, False, False]), ("AB", [False, False, True, False, False]), ("BAB", [False, False, True, False, False]), ("BEG1END", [False, False, False, True, False]), ("BEG1ENDBEG2END", [False, False, False, True, False]), ("y", [False, False, False, False, True]), ("yy", [False, False, False, False, True]), ("yyy", [False, False, False, False, True]), ("invalid", [False, False, False, False, False]), ] @pytest.mark.parametrize("stag_id, stag_format, expected_grammar", plus_stag_grammar) @pytest.mark.parametrize("instance, accepted_results", plus_instance_accepted_results) def test_plus_format( stag_id: int, stag_format: Dict[str, Any], expected_grammar: str, instance: str, accepted_results: List[bool], ): check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, accepted_results[stag_id]) # ---------- StarFormat (0 or more occurrences) ---------- star_stag_grammar = [ ( 0, {"type": "star", "content": {"type": "const_string", "value": "x"}}, r"""const_string ::= (("x")) star ::= ("" | (const_string star)) star_1 ::= ((star)) root ::= ((star_1)) """, ), ( 1, { "type": "star", "content": { "type": "sequence", "elements": [ {"type": "const_string", "value": "a"}, {"type": "const_string", "value": "b"}, ], }, }, r"""const_string ::= (("a")) const_string_1 ::= (("b")) sequence ::= ((const_string const_string_1)) star ::= ("" | (sequence star)) star_1 ::= ((star)) root ::= ((star_1)) """, ), ( 2, { "type": "star", "content": { "type": "or", "elements": [ {"type": "const_string", "value": "A"}, {"type": "const_string", "value": "B"}, ], }, }, r"""const_string ::= (("A")) const_string_1 ::= (("B")) or ::= ((const_string) | (const_string_1)) star ::= ("" | (or star)) star_1 ::= ((star)) root ::= ((star_1)) """, ), ( 3, { "type": "star", "content": { "type": "tag", "begin": "BEG", "content": {"type": "json_schema", "json_schema": {"type": "number"}}, "end": "END", }, }, r"""basic_escape ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) basic_string_sub ::= (("\"") | ([^\0-\x1f\"\\\r\n] basic_string_sub) | ("\\" basic_escape basic_string_sub)) (=([ \n\t]* [,}\]:])) basic_any ::= ((basic_number) | (basic_string) | (basic_boolean) | (basic_null) | (basic_array) | (basic_object)) basic_integer ::= (("0") | (basic_integer_1 [1-9] [0-9]*)) basic_number ::= ((basic_number_1 basic_number_7 basic_number_3 basic_number_6)) basic_string ::= (("\"" basic_string_sub)) basic_boolean ::= (("true") | ("false")) basic_null ::= (("null")) basic_array ::= (("[" [ \n\t]* basic_any basic_array_1 [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= (("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1 [ \n\t]* "}") | ("{" [ \n\t]* "}")) root_0 ::= ((basic_number)) basic_integer_1 ::= ("" | ("-")) basic_number_1 ::= ("" | ("-")) basic_number_2 ::= (([0-9] basic_number_2) | ([0-9])) basic_number_3 ::= ("" | ("." basic_number_2)) basic_number_4 ::= ("" | ([+\-])) basic_number_5 ::= (([0-9] basic_number_5) | ([0-9])) basic_number_6 ::= ("" | ([eE] basic_number_4 basic_number_5)) basic_array_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_any basic_array_1)) basic_object_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1)) basic_number_7 ::= (("0") | ([1-9] [0-9]*)) tag ::= (("BEG" root_0 "END")) star ::= ("" | (tag star)) star_1 ::= ((star)) root ::= ((star_1)) """, ), ( 4, { "type": "star", "content": {"type": "optional", "content": {"type": "const_string", "value": "z"}}, }, r"""const_string ::= (("z")) optional ::= ("" | (const_string)) star ::= ("" | (optional star)) star_1 ::= ((star)) root ::= ((star_1)) """, ), ] star_instance_accepted_results = [ ("", [True, True, True, True, True]), ("x", [True, False, False, False, False]), ("xx", [True, False, False, False, False]), ("xxx", [True, False, False, False, False]), ("ab", [False, True, False, False, False]), ("abab", [False, True, False, False, False]), ("A", [False, False, True, False, False]), ("BAB", [False, False, True, False, False]), ("BEG1END", [False, False, False, True, False]), ("BEG1ENDBEG2END", [False, False, False, True, False]), ("z", [False, False, False, False, True]), ("zz", [False, False, False, False, True]), ("zzz", [False, False, False, False, True]), ("xz", [False, False, False, False, False]), ("invalid", [False, False, False, False, False]), ] @pytest.mark.parametrize("stag_id, stag_format, expected_grammar", star_stag_grammar) @pytest.mark.parametrize("instance, accepted_results", star_instance_accepted_results) def test_star_format( stag_id: int, stag_format: Dict[str, Any], expected_grammar: str, instance: str, accepted_results: List[bool], ): check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, accepted_results[stag_id]) # ---------- RepeatFormat (min to max occurrences) ---------- repeat_stag_grammar = [ # const_string, unbounded (like star) ( 0, {"type": "repeat", "min": 0, "max": -1, "content": {"type": "const_string", "value": "x"}}, r"""const_string ::= (("x")) repeat ::= ((const_string{0, -1})) root ::= ((repeat)) """, ), # const_string, 1+ (like plus) ( 1, {"type": "repeat", "min": 1, "max": -1, "content": {"type": "const_string", "value": "x"}}, r"""const_string ::= (("x")) repeat ::= ((const_string{1, -1})) root ::= ((repeat)) """, ), # const_string, bounded [2, 3] ( 2, {"type": "repeat", "min": 2, "max": 3, "content": {"type": "const_string", "value": "a"}}, r"""const_string ::= (("a")) repeat ::= ((const_string{2, 3})) root ::= ((repeat)) """, ), # const_string, [0, 2] ( 3, {"type": "repeat", "min": 0, "max": 2, "content": {"type": "const_string", "value": "b"}}, r"""const_string ::= (("b")) repeat ::= ((const_string{0, 2})) root ::= ((repeat)) """, ), # sequence content, 1+ unbounded ( 4, { "type": "repeat", "min": 1, "max": -1, "content": { "type": "sequence", "elements": [ {"type": "const_string", "value": "a"}, {"type": "const_string", "value": "b"}, ], }, }, r"""const_string ::= (("a")) const_string_1 ::= (("b")) sequence ::= ((const_string const_string_1)) repeat ::= ((sequence{1, -1})) root ::= ((repeat)) """, ), # or content, [0, 3] ( 5, { "type": "repeat", "min": 0, "max": 3, "content": { "type": "or", "elements": [ {"type": "const_string", "value": "A"}, {"type": "const_string", "value": "B"}, ], }, }, r"""const_string ::= (("A")) const_string_1 ::= (("B")) or ::= ((const_string) | (const_string_1)) repeat ::= ((or{0, 3})) root ::= ((repeat)) """, ), # tag + json_schema content, 0+ unbounded ( 6, { "type": "repeat", "min": 0, "max": -1, "content": { "type": "tag", "begin": "BEG", "content": {"type": "json_schema", "json_schema": {"type": "number"}}, "end": "END", }, }, r"""basic_escape ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) basic_string_sub ::= (("\"") | ([^\0-\x1f\"\\\r\n] basic_string_sub) | ("\\" basic_escape basic_string_sub)) (=([ \n\t]* [,}\]:])) basic_any ::= ((basic_number) | (basic_string) | (basic_boolean) | (basic_null) | (basic_array) | (basic_object)) basic_integer ::= (("0") | (basic_integer_1 [1-9] [0-9]*)) basic_number ::= ((basic_number_1 basic_number_7 basic_number_3 basic_number_6)) basic_string ::= (("\"" basic_string_sub)) basic_boolean ::= (("true") | ("false")) basic_null ::= (("null")) basic_array ::= (("[" [ \n\t]* basic_any basic_array_1 [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= (("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1 [ \n\t]* "}") | ("{" [ \n\t]* "}")) root_0 ::= ((basic_number)) basic_integer_1 ::= ("" | ("-")) basic_number_1 ::= ("" | ("-")) basic_number_2 ::= (([0-9] basic_number_2) | ([0-9])) basic_number_3 ::= ("" | ("." basic_number_2)) basic_number_4 ::= ("" | ([+\-])) basic_number_5 ::= (([0-9] basic_number_5) | ([0-9])) basic_number_6 ::= ("" | ([eE] basic_number_4 basic_number_5)) basic_array_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_any basic_array_1)) basic_object_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any basic_object_1)) basic_number_7 ::= (("0") | ([1-9] [0-9]*)) tag ::= (("BEG" root_0 "END")) repeat ::= ((tag{0, -1})) root ::= ((repeat)) """, ), # optional content, [0, 2] ( 7, { "type": "repeat", "min": 0, "max": 2, "content": {"type": "optional", "content": {"type": "const_string", "value": "y"}}, }, r"""const_string ::= (("y")) optional ::= ("" | (const_string)) repeat ::= ((optional{0, 2})) root ::= ((repeat)) """, ), # const_string, max > 256 (unbounded; -1 already covers “no small cap”) ( 8, {"type": "repeat", "min": 0, "max": -1, "content": {"type": "const_string", "value": "z"}}, r"""const_string ::= (("z")) repeat ::= ((const_string{0, -1})) root ::= ((repeat)) """, ), # const_string, max = 300 (> 128) bounded ( 9, {"type": "repeat", "min": 0, "max": 300, "content": {"type": "const_string", "value": "z"}}, r"""const_string ::= (("z")) repeat ::= ((const_string{0, 300})) root ::= ((repeat)) """, ), # const_string, min=1 max=400 (> 128) ( 10, {"type": "repeat", "min": 1, "max": 400, "content": {"type": "const_string", "value": "w"}}, r"""const_string ::= (("w")) repeat ::= ((const_string{1, 400})) root ::= ((repeat)) """, ), ] repeat_instance_accepted_results = [ # instance -> [accepted for stag 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] ("", [True, False, False, True, False, True, True, True, True, True, False]), ("x", [True, True, False, False, False, False, False, False, False, False, False]), ("xx", [True, True, False, False, False, False, False, False, False, False, False]), ("xxx", [True, True, False, False, False, False, False, False, False, False, False]), ("a", [False, False, False, False, False, False, False, False, False, False, False]), ("aa", [False, False, True, False, False, False, False, False, False, False, False]), ("aaa", [False, False, True, False, False, False, False, False, False, False, False]), ("aaaa", [False, False, False, False, False, False, False, False, False, False, False]), ("b", [False, False, False, True, False, False, False, False, False, False, False]), ("bb", [False, False, False, True, False, False, False, False, False, False, False]), ("bbb", [False, False, False, False, False, False, False, False, False, False, False]), ("ab", [False, False, False, False, True, False, False, False, False, False, False]), ("abab", [False, False, False, False, True, False, False, False, False, False, False]), ("A", [False, False, False, False, False, True, False, False, False, False, False]), ("B", [False, False, False, False, False, True, False, False, False, False, False]), ("AB", [False, False, False, False, False, True, False, False, False, False, False]), ("AAB", [False, False, False, False, False, True, False, False, False, False, False]), ("AABA", [False, False, False, False, False, False, False, False, False, False, False]), ("AAAB", [False, False, False, False, False, False, False, False, False, False, False]), ("BEG1END", [False, False, False, False, False, False, True, False, False, False, False]), ( "BEG1ENDBEG2END", [False, False, False, False, False, False, True, False, False, False, False], ), ("y", [False, False, False, False, False, False, False, True, False, False, False]), ("yy", [False, False, False, False, False, False, False, True, False, False, False]), ("yyy", [False, False, False, False, False, False, False, False, False, False, False]), ("z", [False, False, False, False, False, False, False, False, True, True, False]), ("zz", [False, False, False, False, False, False, False, False, True, True, False]), ("z" * 100, [False, False, False, False, False, False, False, False, True, True, False]), ("z" * 350, [False, False, False, False, False, False, False, False, True, False, False]), ("w", [False, False, False, False, False, False, False, False, False, False, True]), ("ww", [False, False, False, False, False, False, False, False, False, False, True]), ("w" * 100, [False, False, False, False, False, False, False, False, False, False, True]), ("w" * 450, [False, False, False, False, False, False, False, False, False, False, False]), ("invalid", [False, False, False, False, False, False, False, False, False, False, False]), ] @pytest.mark.parametrize("stag_id, stag_format, expected_grammar", repeat_stag_grammar) @pytest.mark.parametrize("instance, accepted_results", repeat_instance_accepted_results) def test_repeat_format( stag_id: int, stag_format: Dict[str, Any], expected_grammar: str, instance: str, accepted_results: List[bool], ): check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, accepted_results[stag_id]) compound_stag_instance_is_accepted = [ # Llama JSON-based tool calling ( { "type": "triggered_tags", "triggers": ['{"name":'], "tags": [ { "begin": '{"name": "func1", "parameters": ', "content": {"type": "json_schema", "json_schema": {"type": "object"}}, "end": "}", }, { "begin": '{"name": "func2", "parameters": ', "content": {"type": "json_schema", "json_schema": {"type": "object"}}, "end": "}", }, ], }, [ ( '{"name": "func2", "parameters": {"arg": 10}}{"name": "func1", "parameters": {"arg": "123"}}', True, ), ('{"name": "func3", "parameters": {"arg": 10}}', False), ], ), # Force think ( { "type": "sequence", "elements": [ { "type": "tag", "begin": "", "content": {"type": "any_text"}, "end": "", }, { "type": "triggered_tags", "triggers": ["", "content": {"type": "json_schema", "json_schema": {"type": "object"}}, "end": "", }, { "begin": "", "content": {"type": "json_schema", "json_schema": {"type": "object"}}, "end": "", }, ], }, ], }, [ ( '[any_text][any_text]{"arg": 10}[any_text]{"arg": 10}[any_text]', True, ), ( '[any_text]{"arg": 10}[any_text]{"arg": 10}[any_text]', False, ), ('[any_text][any_text]{"arg": 10}', False), ], ), # Think & Force tool calling (Llama style) ( { "type": "sequence", "elements": [ { "type": "tag", "begin": "", "content": {"type": "any_text"}, "end": "", }, { "type": "triggered_tags", "triggers": ["", "content": {"type": "json_schema", "json_schema": {"type": "object"}}, "end": "", }, { "begin": "", "content": {"type": "json_schema", "json_schema": {"type": "object"}}, "end": "", }, ], "stop_after_first": True, "at_least_one": True, }, ], }, [ ('[any_text]{"arg": 10}', True), ('[any_text][any_text]{"arg": 10}', False), ('[any_text]{"arg": 10}[any_text]', False), ], ), # Think & force tool calling (DeepSeek style) ( { "type": "sequence", "elements": [ { "type": "tag", "begin": "", "content": {"type": "any_text"}, "end": "", }, { "type": "triggered_tags", "triggers": ["<｜tool▁calls▁begin｜>"], "tags": [ { "begin": "<｜tool▁calls▁begin｜>", "end": "<｜tool▁calls▁end｜>", "content": { "type": "tags_with_separator", "separator": "\n", "tags": [ { "begin": "<｜tool▁call▁begin｜>function<｜tool▁sep｜>function_name_1\n```json\n", "content": { "type": "json_schema", "json_schema": {"type": "object"}, }, "end": "\n```<｜tool▁call▁end｜>", }, { "begin": "<｜tool▁call▁begin｜>function<｜tool▁sep｜>function_name_2\n```json\n", "content": { "type": "json_schema", "json_schema": {"type": "object"}, }, "end": "\n```<｜tool▁call▁end｜>", }, ], }, } ], "stop_after_first": True, }, ], }, [ ("[any_text][any_text]", True), ("[any_text][any_text]<｜tool▁calls▁begin｜><｜tool▁calls▁end｜>", True), ( """[any_text][any_text]<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>function_name_1 ```json {"arg": 10} ```<｜tool▁call▁end｜> <｜tool▁call▁begin｜>function<｜tool▁sep｜>function_name_2 ```json {"arg": 10} ```<｜tool▁call▁end｜><｜tool▁calls▁end｜>""", True, ), ( """[any_text][any_text]<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>function_name_3 ```json {"arg": 10} ```<｜tool▁call▁end｜><｜tool▁calls▁end｜>""", False, ), ( """[any_text][any_text]<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>function_name_2 ```json {"arg": 10} ```<｜tool▁call▁end｜><｜tool▁calls▁end｜>[any_text]""", False, ), ], ), # Force non-think mode ( { "type": "sequence", "elements": [ {"type": "const_string", "value": ""}, { "type": "triggered_tags", "triggers": [""], "tags": [ { "begin": '\n{"name": "func1", "arguments": ', "content": {"type": "json_schema", "json_schema": {"type": "object"}}, "end": "}\n", }, { "begin": '\n{"name": "func2", "arguments": ', "content": {"type": "json_schema", "json_schema": {"type": "object"}}, "end": "}\n", }, ], }, ], }, [ ( '[any_text]\n{"name": "func1", "arguments": {"arg": 10}}\n[any_text]', True, ), ( 'abcd[any_text]\n{"name": "func1", "arguments": {"arg": 10}}\n[any_text]', False, ), ], ), ] @pytest.mark.parametrize( "stag_format, instance_is_accepted_tuples", compound_stag_instance_is_accepted ) def test_compound_format( stag_format: Dict[str, Any], instance_is_accepted_tuples: List[Tuple[str, bool]] ): for instance, is_accepted in instance_is_accepted_tuples: check_stag_with_instance(stag_format, instance, is_accepted) end_string_detector_test_data = [ ( { "type": "tag", "begin": "", "content": { "type": "sequence", "elements": [{"type": "const_string", "value": "[TEXT]"}, {"type": "any_text"}], }, "end": "", }, r"""const_string ::= (("[TEXT]")) any_text ::= TagDispatch( loop_after_dispatch=false, excludes=("") ) sequence ::= ((const_string any_text)) tag ::= (("" sequence "")) root ::= ((tag)) """, [ ("[TEXT]", True), ("[TEXT]abcde", True), ("[TEXT]abcde", False), ("", False), ], ), ( # Detect the end string for nested structures { "type": "tag", "begin": "", "content": { "type": "or", "elements": [ { "type": "triggered_tags", "triggers": ["", "content": {"type": "any_text"}, "end": ""} ], "at_least_one": True, }, { "type": "sequence", "elements": [ {"type": "const_string", "value": "[TEXT2]"}, {"type": "any_text"}, ], }, { "type": "tags_with_separator", "tags": [ {"begin": "", "content": {"type": "any_text"}, "end": ""} ], "separator": "", }, ], }, "end": "", }, r"""any_text ::= TagDispatch( loop_after_dispatch=false, excludes=("") ) triggered_tags_group ::= ((">" any_text "")) triggered_tags_first ::= (("" any_text "")) triggered_tags_sub ::= TagDispatch( ("") ) triggered_tags ::= ((triggered_tags_first triggered_tags_sub)) const_string ::= (("[TEXT2]")) any_text_1 ::= TagDispatch( loop_after_dispatch=false, excludes=("") ) sequence ::= ((const_string any_text_1)) any_text_2 ::= TagDispatch( loop_after_dispatch=false, excludes=("") ) tag ::= (("" any_text_2 "")) tags_with_separator_tags ::= ((tag)) tags_with_separator_sub ::= ("" | ("" tags_with_separator_tags tags_with_separator_sub)) tags_with_separator ::= ("" | (tags_with_separator_tags tags_with_separator_sub)) or ::= ((triggered_tags) | (sequence) | (tags_with_separator)) tag_1 ::= (("" or "")) root ::= ((tag_1)) """, [ ("[TEXT]", True), ("", True), ("[TEXT2]abc", True), ("abc", True), ("", True), ("", True), ("[TEXT2]", False), ], ), ( # Also in nested structures, but none end string can be detected { "type": "or", "elements": [ { "type": "triggered_tags", "triggers": ["", "content": {"type": "any_text"}, "end": ""} ], "at_least_one": True, }, { "type": "sequence", "elements": [{"type": "const_string", "value": "[TEXT]"}, {"type": "any_text"}], }, { "type": "or", "elements": [ { "type": "tags_with_separator", "tags": [ { "begin": "", "content": {"type": "any_text"}, "end": "", } ], "separator": "", "at_least_one": True, }, { "type": "sequence", "elements": [ {"type": "const_string", "value": "[TEXT2]"}, {"type": "any_text"}, ], }, ], }, ], }, r"""any_text ::= TagDispatch( loop_after_dispatch=false, excludes=("") ) triggered_tags_group ::= ((">" any_text "")) triggered_tags_first ::= (("" any_text "")) triggered_tags_sub ::= TagDispatch( ("") ) tag ::= (("" any_text_2 "")) tags_with_separator_tags ::= ((tag)) tags_with_separator_sub ::= ("" | ("" tags_with_separator_tags tags_with_separator_sub)) tags_with_separator ::= ((tags_with_separator_tags tags_with_separator_sub)) const_string_1 ::= (("[TEXT2]")) sequence_1 ::= ((const_string_1 any_text_1)) or ::= ((tags_with_separator) | (sequence_1)) or_1 ::= ((triggered_tags) | (sequence) | (or)) root ::= ((or_1)) """, [ ("abcabcdef", True), ("[TEXT]abc", True), ("[TEXT]", True), ("abc", True), ("abcdef", True), ("[TEXT2]def", True), ("[TEXT2]", True), ("abc", False), ("abc", False), ("abc", False), ("abc", False), ("abcdef", False), ("random text", False), ], ), ] @pytest.mark.parametrize( "stag_format, expected_grammar, instance_is_accepted_tuples", end_string_detector_test_data ) def test_end_string_detector( stag_format: Dict[str, Any], expected_grammar: str, instance_is_accepted_tuples: List[Tuple[str, bool]], ): check_stag_with_grammar(stag_format, expected_grammar) for instance, is_accepted in instance_is_accepted_tuples: check_stag_with_instance(stag_format, instance, is_accepted) # Test cases for JSON format and parsing errors (need string input) json_format_error_test_data = [ # JSON Parsing Errors ( '{"type": "structural_tag", "format": {"type": "const_string", "value": "hello"', "Failed to parse JSON", ), ('"not_an_object"', "Structural tag must be an object"), ( '{"type": "wrong_type", "format": {"type": "const_string", "value": "hello"}}', 'Structural tag\'s type must be a string "structural_tag"', ), ('{"type": "structural_tag"}', "Structural tag must have a format field"), # Format Parsing Errors ('{"type": "structural_tag", "format": "not_an_object"}', "Format must be an object"), ( '{"type": "structural_tag", "format": {"type": 123, "value": "hello"}}', "Format's type must be a string", ), ( '{"type": "structural_tag", "format": {"type": "unknown_format"}}', "Format type not recognized: unknown_format", ), ('{"type": "structural_tag", "format": {"invalid_field": "value"}}', "Invalid format"), # ConstStringFormat Errors ( '{"type": "structural_tag", "format": {"type": "const_string"}}', "ConstString format must have a value field with a string", ), ( '{"type": "structural_tag", "format": {"type": "const_string", "value": 123}}', "ConstString format must have a value field with a string", ), # JSONSchemaFormat Errors ( '{"type": "structural_tag", "format": {"type": "json_schema"}}', "JSON schema format must have a json_schema field with a object or boolean value", ), ( '{"type": "structural_tag", "format": {"type": "json_schema", "json_schema": "invalid"}}', "JSON schema format must have a json_schema field with a object or boolean value", ), # SequenceFormat Errors ( '{"type": "structural_tag", "format": {"type": "sequence"}}', "Sequence format must have an elements field with an array", ), ( '{"type": "structural_tag", "format": {"type": "sequence", "elements": "not_array"}}', "Sequence format must have an elements field with an array", ), ( '{"type": "structural_tag", "format": {"type": "sequence", "elements": []}}', "Sequence format must have at least one element", ), # OrFormat Errors ( '{"type": "structural_tag", "format": {"type": "or"}}', "Or format must have an elements field with an array", ), ( '{"type": "structural_tag", "format": {"type": "or", "elements": "not_array"}}', "Or format must have an elements field with an array", ), ( '{"type": "structural_tag", "format": {"type": "or", "elements": []}}', "Or format must have at least one element", ), # TagFormat Errors ( '{"type": "structural_tag", "format": {"type": "tag", "content": {"type": "const_string", "value": "hello"}, "end": "end"}}', "Tag format's begin field must be a string", ), ( '{"type": "structural_tag", "format": {"type": "tag", "begin": 123, "content": {"type": "const_string", "value": "hello"}, "end": "end"}}', "Tag format's begin field must be a string", ), ( '{"type": "structural_tag", "format": {"type": "tag", "begin": "start", "end": "end"}}', "Tag format must have a content field", ), ( '{"type": "structural_tag", "format": {"type": "tag", "begin": "start", "content": {"type": "const_string", "value": "hello"}}}', "Tag format must have an end field", ), ( '{"type": "structural_tag", "format": {"type": "tag", "begin": "start", "content": {"type": "const_string", "value": "hello"}, "end": 123}}', "Tag format's end field must be a string or array of strings", ), # TriggeredTagsFormat Errors ( '{"type": "structural_tag", "format": {"type": "triggered_tags", "tags": [{"begin": "start", "content": {"type": "const_string", "value": "hello"}, "end": "end"}]}}', "Triggered tags format must have a triggers field with an array", ), ( '{"type": "structural_tag", "format": {"type": "triggered_tags", "triggers": "not_array", "tags": [{"begin": "start", "content": {"type": "const_string", "value": "hello"}, "end": "end"}]}}', "Triggered tags format must have a triggers field with an array", ), ( '{"type": "structural_tag", "format": {"type": "triggered_tags", "triggers": [], "tags": [{"begin": "start", "content": {"type": "const_string", "value": "hello"}, "end": "end"}]}}', "Triggered tags format's triggers must be non-empty", ), ( '{"type": "structural_tag", "format": {"type": "triggered_tags", "triggers": [123], "tags": [{"begin": "start", "content": {"type": "const_string", "value": "hello"}, "end": "end"}]}}', "Triggered tags format's triggers must be non-empty strings", ), ( '{"type": "structural_tag", "format": {"type": "triggered_tags", "triggers": [""], "tags": [{"begin": "start", "content": {"type": "const_string", "value": "hello"}, "end": "end"}]}}', "Triggered tags format's triggers must be non-empty strings", ), ( '{"type": "structural_tag", "format": {"type": "triggered_tags", "triggers": ["trigger"]}}', "Triggered tags format must have a tags field with an array", ), ( '{"type": "structural_tag", "format": {"type": "triggered_tags", "triggers": ["trigger"], "tags": "not_array"}}', "Triggered tags format must have a tags field with an array", ), ( '{"type": "structural_tag", "format": {"type": "triggered_tags", "triggers": ["trigger"], "tags": []}}', "Triggered tags format's tags must be non-empty", ), ( '{"type": "structural_tag", "format": {"type": "triggered_tags", "triggers": ["trigger"], "tags": [{"begin": "start", "content": {"type": "const_string", "value": "hello"}, "end": "end"}], "at_least_one": "not_boolean"}}', "at_least_one must be a boolean", ), ( '{"type": "structural_tag", "format": {"type": "triggered_tags", "triggers": ["trigger"], "tags": [{"begin": "start", "content": {"type": "const_string", "value": "hello"}, "end": "end"}], "stop_after_first": "not_boolean"}}', "stop_after_first must be a boolean", ), # TagsWithSeparatorFormat Errors ( '{"type": "structural_tag", "format": {"type": "tags_with_separator", "separator": "sep"}}', "Tags with separator format must have a tags field with an array", ), ( '{"type": "structural_tag", "format": {"type": "tags_with_separator", "tags": "not_array", "separator": "sep"}}', "Tags with separator format must have a tags field with an array", ), ( '{"type": "structural_tag", "format": {"type": "tags_with_separator", "tags": [], "separator": "sep"}}', "Tags with separator format's tags must be non-empty", ), ( '{"type": "structural_tag", "format": {"type": "tags_with_separator", "tags": [{"begin": "start", "content": {"type": "const_string", "value": "hello"}, "end": "end"}]}}', "Tags with separator format's separator field must be a string", ), ( '{"type": "structural_tag", "format": {"type": "tags_with_separator", "tags": [{"begin": "start", "content": {"type": "const_string", "value": "hello"}, "end": "end"}], "separator": 123}}', "Tags with separator format's separator field must be a string", ), # Note: empty separator is now valid, so no error test for it ( '{"type": "structural_tag", "format": {"type": "tags_with_separator", "tags": [{"begin": "start", "content": {"type": "const_string", "value": "hello"}, "end": "end"}], "separator": "sep", "at_least_one": "not_boolean"}}', "at_least_one must be a boolean", ), ( '{"type": "structural_tag", "format": {"type": "tags_with_separator", "tags": [{"begin": "start", "content": {"type": "const_string", "value": "hello"}, "end": "end"}], "separator": "sep", "stop_after_first": "not_boolean"}}', "stop_after_first must be a boolean", ), ( '{"type": "structural_tag", "format": {"type": "json_schema", "json_schema": {"type": "string"}, "style": "not_string"}}', 'style must be "json", "qwen_xml", "minimax_xml", "deepseek_xml", or "glm_xml"', ), # RepeatFormat Errors - illegal min/max ( '{"type": "structural_tag", "format": {"type": "repeat", "min": -1, "max": 5, "content": {"type": "const_string", "value": "x"}}}', "Repeat min must be >= 0", ), ( '{"type": "structural_tag", "format": {"type": "repeat", "min": 5, "max": 3, "content": {"type": "const_string", "value": "x"}}}', "Repeat min must be <= max", ), ( '{"type": "structural_tag", "format": {"type": "repeat", "min": 0, "max": -2, "content": {"type": "const_string", "value": "x"}}}', "Repeat max must be -1 (unbounded) or >= 0", ), ] @pytest.mark.parametrize("json_input, expected_error", json_format_error_test_data) def test_structural_tag_json_format_errors(json_input: str, expected_error: str): """Test JSON format and parsing errors that occur during JSON parsing phase""" with pytest.raises(Exception) as exc_info: xgr.Grammar.from_structural_tag(json_input) assert expected_error in str(exc_info.value) structural_tag_error_test_data = [ # Analyzer Errors - Tag format with unlimited content but empty end { "type": "tag", "begin": "start", "content": {"type": "any_text"}, # Unlimited content "end": "", # Empty end with unlimited content causes error }, # Converter Errors - Tag matches multiple triggers { "type": "triggered_tags", "triggers": ["A", "AB"], # Both will match tag beginning with "ABC" "tags": [ {"begin": "ABC", "content": {"type": "const_string", "value": "hello"}, "end": "end"} ], }, # Converter Errors - Tag matches no trigger { "type": "triggered_tags", "triggers": ["X", "Y"], # Neither matches "ABC" begin "tags": [ {"begin": "ABC", "content": {"type": "const_string", "value": "hello"}, "end": "end"} ], }, # Original test cases - Detected end string of tags_with_separator is empty { "type": "tag", "begin": "", "content": { "type": "tags_with_separator", "tags": [ { "begin": "", "content": {"type": "const_string", "value": "[TEXT]"}, "end": "", } ], "separator": "", }, "end": "", }, ] @pytest.mark.parametrize("stag_format", structural_tag_error_test_data) def test_structural_tag_error(stag_format: Dict[str, Any]): """Test analyzer and converter errors that occur after successful parsing""" structural_tag = {"type": "structural_tag", "format": stag_format} with pytest.raises(Exception, match="Invalid structural tag error"): xgr.Grammar.from_structural_tag(structural_tag) utf8_stag_format_and_instance_accepted = [ ({"type": "const_string", "value": "你好"}, "你好", True), ({"type": "const_string", "value": "你好"}, "hello", False), ({"type": "any_text"}, "😊", True), ( { "type": "sequence", "elements": [ {"type": "const_string", "value": "开始"}, {"type": "json_schema", "json_schema": {"type": "string"}}, {"type": "const_string", "value": "结束"}, ], }, '开始"中间"结束', True, ), ( { "type": "sequence", "elements": [ {"type": "const_string", "value": "开始"}, {"type": "json_schema", "json_schema": {"type": "string"}}, {"type": "const_string", "value": "结束"}, ], }, "开始中间内容", False, ), ( {"type": "tag", "begin": "标签开始", "content": {"type": "any_text"}, "end": "标签结束"}, "标签开始一些内容标签结束", True, ), ( {"type": "tag", "begin": "标签开始", "content": {"type": "any_text"}, "end": "标签结束"}, "标签开始一些内容", False, ), ( { "type": "or", "elements": [ {"type": "const_string", "value": "选项一"}, {"type": "const_string", "value": "选项二"}, ], }, "选项一", True, ), ( { "type": "or", "elements": [ {"type": "const_string", "value": "选项一"}, {"type": "const_string", "value": "选项二"}, ], }, "选项三", False, ), ( { "type": "tags_with_separator", "tags": [{"begin": "项开始", "content": {"type": "any_text"}, "end": "项结束"}], "separator": "分隔符", }, "项开始内容1项结束分隔符项开始内容2项结束", True, ), ( { "type": "tags_with_separator", "tags": [{"begin": "项开始", "content": {"type": "any_text"}, "end": "项结束"}], "separator": "分隔符", }, "项开始内容1项结束项开始内容2项结束", False, ), ( { "type": "json_schema", "json_schema": { "type": "object", "properties": {"字段": {"type": "string"}}, "required": ["字段"], "additionalProperties": False, }, }, '{"字段": "值"}', True, ), ( { "type": "qwen_xml_parameter", "json_schema": { "type": "object", "properties": {"参数": {"type": "string"}}, "required": ["参数"], "additionalProperties": False, }, }, "值", True, ), ] @pytest.mark.parametrize( "stag_format, instance, is_accepted", utf8_stag_format_and_instance_accepted ) def test_basic_structural_tag_utf8(stag_format: Dict[str, Any], instance: str, is_accepted: bool): """Test structural tag with UTF-8 characters""" check_stag_with_instance(stag_format, instance, is_accepted) basic_structural_tags_instance_is_accepted = [ # ConstStringFormat (xgr.structural_tag.ConstStringFormat(value="hello"), "hello", True), (xgr.structural_tag.ConstStringFormat(value="hello"), "hello world", False), # JSONSchemaFormat (xgr.structural_tag.JSONSchemaFormat(json_schema={"type": "object"}), '{"key": "value"}', True), (xgr.structural_tag.JSONSchemaFormat(json_schema={"type": "string"}), '"abc"', True), (xgr.structural_tag.JSONSchemaFormat(json_schema={"type": "integer"}), "123", True), (xgr.structural_tag.JSONSchemaFormat(json_schema={"type": "integer"}), "abc", False), # JSONSchemaFormat with style="qwen_xml" ( xgr.structural_tag.JSONSchemaFormat( json_schema={"type": "object", "properties": {"name": {"type": "string"}}}, style="qwen_xml", ), "value", True, ), ( xgr.structural_tag.JSONSchemaFormat( json_schema={"type": "object", "properties": {"name": {"type": "string"}}}, style="qwen_xml", ), "value", False, ), # JSONSchemaFormat with style="minimax_xml" ( xgr.structural_tag.JSONSchemaFormat( json_schema={"type": "object", "properties": {"name": {"type": "string"}}}, style="minimax_xml", ), 'value', True, ), ( xgr.structural_tag.JSONSchemaFormat( json_schema={"type": "object", "properties": {"name": {"type": "string"}}}, style="minimax_xml", ), 'value', False, ), # JSONSchemaFormat with style="deepseek_xml" ( xgr.structural_tag.JSONSchemaFormat( json_schema={"type": "object", "properties": {"name": {"type": "string"}}}, style="deepseek_xml", ), '<｜DSML｜parameter name="name" string="true">value', True, ), ( xgr.structural_tag.JSONSchemaFormat( json_schema={"type": "object", "properties": {"name": {"type": "string"}}}, style="deepseek_xml", ), '<｜DSML｜parameter name="name" string="true">value', False, ), # JSONSchemaFormat with style="glm_xml" ( xgr.structural_tag.JSONSchemaFormat( json_schema={"type": "object", "properties": {"name": {"type": "string"}}}, style="glm_xml", ), "namevalue", True, ), ( xgr.structural_tag.JSONSchemaFormat( json_schema={"type": "object", "properties": {"name": {"type": "string"}}}, style="glm_xml", ), "namevalue", False, ), # AnyTextFormat (xgr.structural_tag.AnyTextFormat(), "", True), (xgr.structural_tag.AnyTextFormat(), "any text here", True), # SequenceFormat ( xgr.structural_tag.SequenceFormat( elements=[ xgr.structural_tag.ConstStringFormat(value="A"), xgr.structural_tag.ConstStringFormat(value="B"), ] ), "AB", True, ), ( xgr.structural_tag.SequenceFormat( elements=[ xgr.structural_tag.ConstStringFormat(value="A"), xgr.structural_tag.ConstStringFormat(value="B"), ] ), "A", False, ), # OrFormat ( xgr.structural_tag.OrFormat( elements=[ xgr.structural_tag.ConstStringFormat(value="A"), xgr.structural_tag.ConstStringFormat(value="B"), ] ), "A", True, ), ( xgr.structural_tag.OrFormat( elements=[ xgr.structural_tag.ConstStringFormat(value="A"), xgr.structural_tag.ConstStringFormat(value="B"), ] ), "B", True, ), ( xgr.structural_tag.OrFormat( elements=[ xgr.structural_tag.ConstStringFormat(value="A"), xgr.structural_tag.ConstStringFormat(value="B"), ] ), "C", False, ), # TagFormat ( xgr.structural_tag.TagFormat( begin="", content=xgr.structural_tag.AnyTextFormat(), end="" ), "text", True, ), ( xgr.structural_tag.TagFormat( begin="", content=xgr.structural_tag.AnyTextFormat(), end="" ), "text"1","2"', True, ), ( xgr.structural_tag.TagsWithSeparatorFormat( tags=[ xgr.structural_tag.TagFormat( begin="", content=xgr.structural_tag.AnyTextFormat(), end="" ) ], separator=",", ), '"1""2"', False, ), # QwenXMLParameterFormat ( xgr.structural_tag.QwenXMLParameterFormat( json_schema={"type": "object", "properties": {"name": {"type": "string"}}} ), "value", True, ), ( xgr.structural_tag.QwenXMLParameterFormat( json_schema={"type": "object", "properties": {"name": {"type": "string"}}} ), "value", False, ), ] @pytest.mark.parametrize( "stag_format, instance, is_accepted", basic_structural_tags_instance_is_accepted ) def test_from_structural_tag_with_structural_tag_instance( stag_format: xgr.structural_tag.Format, instance: str, is_accepted: bool ): stag = xgr.StructuralTag(format=stag_format) check_stag_with_instance(stag, instance, is_accepted) # ---------- Multiple End Tokens Tests ---------- multiple_end_tokens_tag_stag_grammar = [ # Test tag with multiple end tokens (limited content) ( { "type": "tag", "begin": "BEG", "content": {"type": "const_string", "value": "CONTENT"}, "end": ["END1", "END2"], }, r"""const_string ::= (("CONTENT")) tag_end ::= (("END1") | ("END2")) tag ::= (("BEG" const_string tag_end)) root ::= ((tag)) """, ), # Test tag with single end token in array (should work the same as string) ( { "type": "tag", "begin": "", "content": {"type": "const_string", "value": "X"}, "end": [""], }, r"""const_string ::= (("X")) tag ::= (("" const_string "")) root ::= ((tag)) """, ), ] multiple_end_tokens_instance_is_accepted = [ ("BEGCONTENTEND1", True), ("BEGCONTENTEND2", True), ("BEGCONTENTEND3", False), ("BEGCONTENTEND", False), ] @pytest.mark.parametrize("stag_format, expected_grammar", multiple_end_tokens_tag_stag_grammar) def test_multiple_end_tokens_tag_grammar(stag_format: Dict[str, Any], expected_grammar: str): check_stag_with_grammar(stag_format, expected_grammar) @pytest.mark.parametrize("instance, is_accepted", multiple_end_tokens_instance_is_accepted) def test_multiple_end_tokens_tag_instance(instance: str, is_accepted: bool): stag_format = { "type": "tag", "begin": "BEG", "content": {"type": "const_string", "value": "CONTENT"}, "end": ["END1", "END2"], } check_stag_with_instance(stag_format, instance, is_accepted) # Test multiple end tokens with any_text (unlimited content) multiple_end_tokens_any_text_stag_grammar = [ ( {"type": "tag", "begin": "BEG", "content": {"type": "any_text"}, "end": ["END1", "END2"]}, r"""any_text ::= TagDispatch( loop_after_dispatch=false, excludes=("END1", "END2") ) tag_end ::= (("END1") | ("END2")) tag ::= (("BEG" any_text tag_end)) root ::= ((tag)) """, ) ] multiple_end_tokens_any_text_instance_is_accepted = [ ("BEGHello!END1", True), ("BEGHello!END2", True), ("BEGEND1", True), ("BEGEND2", True), ("BEGsome text hereEND1", True), ("BEGsome text hereEND2", True), ("BEGHello!END3", False), ("BEGHello!END", False), ] @pytest.mark.parametrize("stag_format, expected_grammar", multiple_end_tokens_any_text_stag_grammar) def test_multiple_end_tokens_any_text_grammar(stag_format: Dict[str, Any], expected_grammar: str): check_stag_with_grammar(stag_format, expected_grammar) @pytest.mark.parametrize("instance, is_accepted", multiple_end_tokens_any_text_instance_is_accepted) def test_multiple_end_tokens_any_text_instance(instance: str, is_accepted: bool): stag_format = { "type": "tag", "begin": "BEG", "content": {"type": "any_text"}, "end": ["END1", "END2"], } check_stag_with_instance(stag_format, instance, is_accepted) # Test multiple end tokens with one empty string multiple_end_tokens_with_empty_stag_grammar = [ # Test tag with one actual end token and one empty string ( { "type": "tag", "begin": "BEG", "content": {"type": "const_string", "value": "CONTENT"}, "end": ["END1", ""], }, r"""const_string ::= (("CONTENT")) tag_end ::= ("" | ("END1")) tag ::= (("BEG" const_string tag_end)) root ::= ((tag)) """, ), # Test with empty string first ( { "type": "tag", "begin": "", "content": {"type": "const_string", "value": "X"}, "end": ["", ""], }, r"""const_string ::= (("X")) tag_end ::= ("" | ("")) tag ::= (("" const_string tag_end)) root ::= ((tag)) """, ), ] multiple_end_tokens_with_empty_instance_is_accepted = [ ("BEGCONTENTEND1", True), # Ends with END1 ("BEGCONTENT", True), # Ends with empty string ("BEGCONTENTEND2", False), # Wrong end token ("BEGCONTENTEND", False), # Partial match of END1 ] @pytest.mark.parametrize( "stag_format, expected_grammar", multiple_end_tokens_with_empty_stag_grammar ) def test_multiple_end_tokens_with_empty_grammar(stag_format: Dict[str, Any], expected_grammar: str): check_stag_with_grammar(stag_format, expected_grammar) @pytest.mark.parametrize( "instance, is_accepted", multiple_end_tokens_with_empty_instance_is_accepted ) def test_multiple_end_tokens_with_empty_instance(instance: str, is_accepted: bool): stag_format = { "type": "tag", "begin": "BEG", "content": {"type": "const_string", "value": "CONTENT"}, "end": ["END1", ""], } check_stag_with_instance(stag_format, instance, is_accepted) # Test multiple end tokens with Python API def test_multiple_end_tokens_python_api(): """Test that TagFormat accepts both str and List[str] for end field""" # Test with single string (backward compatible) tag1 = xgr.structural_tag.TagFormat( begin="", content=xgr.structural_tag.ConstStringFormat(value="content"), end="" ) assert tag1.end == "" # Test with list of strings tag2 = xgr.structural_tag.TagFormat( begin="", content=xgr.structural_tag.ConstStringFormat(value="content"), end=["", ""], ) assert tag2.end == ["", ""] # Test that both work in StructuralTag stag1 = xgr.StructuralTag(format=tag1) stag2 = xgr.StructuralTag(format=tag2) # Test that the grammars can be created grammar1 = xgr.Grammar.from_structural_tag(stag1) grammar2 = xgr.Grammar.from_structural_tag(stag2) assert grammar1 is not None assert grammar2 is not None # Test error case: empty end array def test_multiple_end_tokens_empty_array_error(): """Test that empty end array raises an error""" stag_format = { "type": "structural_tag", "format": { "type": "tag", "begin": "BEG", "content": {"type": "const_string", "value": "X"}, "end": [], }, } with pytest.raises(Exception) as exc_info: xgr.Grammar.from_structural_tag(stag_format) assert "empty" in str(exc_info.value).lower() # Test error case: unlimited content with all empty end strings def test_multiple_end_tokens_unlimited_empty_error(): """Test that unlimited content with all empty end strings raises an error""" stag_format = { "type": "structural_tag", "format": {"type": "tag", "begin": "BEG", "content": {"type": "any_text"}, "end": ["", ""]}, } with pytest.raises(Exception) as exc_info: xgr.Grammar.from_structural_tag(stag_format) assert "non-empty" in str(exc_info.value).lower() or "empty" in str(exc_info.value).lower() # ---------- Excludes Tests ---------- test_strings_is_accepted_any_text_excludes = [ ("This is a test string.", True), ("This string contains which is excluded.", False), ("Another string with inside.", False), ("A clean string without excluded substrings.", True), (" at the beginning.", False), ("At the end .", False), ] @pytest.mark.parametrize("instance, is_accepted", test_strings_is_accepted_any_text_excludes) def test_excluded_strings_in_any_text(instance: str, is_accepted: bool): stag_format = { "type": "tag", "content": {"type": "any_text", "excludes": ["", ""]}, "begin": "", "end": ".", } expected_grammar = r"""any_text ::= TagDispatch( loop_after_dispatch=false, excludes=("", "", ".") ) tag ::= (("" any_text ".")) root ::= ((tag)) """ check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, is_accepted) test_strings_is_accepted_triggered_excludes = [ ("A", False), ("A1", False), ("A1L1AB", True), ("A1L2A", False), ("L1A1L1A", False), ("L2A2L2A", False), ("A1L1AL1", False), ("A1L1AA2L2A", True), ] @pytest.mark.parametrize("instance, is_accepted", test_strings_is_accepted_triggered_excludes) def test_excluded_strings_in_triggered_format(instance: str, is_accepted: bool): stag_format = { "type": "triggered_tags", "triggers": ["A"], "tags": [ {"begin": "A1", "content": {"type": "const_string", "value": "L1"}, "end": "A"}, {"begin": "A2", "content": {"type": "const_string", "value": "L2"}, "end": "A"}, ], "at_least_one": True, "stop_after_first": False, "excludes": ["L1", "L2"], } expected_grammar = r"""const_string ::= (("L1")) const_string_1 ::= (("L2")) triggered_tags_group ::= (("1" const_string "A") | ("2" const_string_1 "A")) triggered_tags_first ::= (("A1" const_string "A") | ("A2" const_string_1 "A")) triggered_tags_sub ::= TagDispatch( ("A", triggered_tags_group), loop_after_dispatch=true, excludes=("L1", "L2") ) triggered_tags ::= ((triggered_tags_first triggered_tags_sub)) root ::= ((triggered_tags)) """ check_stag_with_grammar(stag_format, expected_grammar) check_stag_with_instance(stag_format, instance, is_accepted) test_strings_is_accepted_single_excludes = [ ("XYZ", True), ("Hello World", True), ("ABC", False), ("123ABC456", False), ("A quick brown fox", True), ("", True), ] @pytest.mark.parametrize("instance, is_accepted", test_strings_is_accepted_single_excludes) def test_excluded_strings_in_single_any_text(instance: str, is_accepted: bool): format = {"type": "any_text", "excludes": ["ABC"]} expected_grammar = r"""any_text ::= TagDispatch( loop_after_dispatch=false, excludes=("ABC") ) root ::= ((any_text)) """ check_stag_with_grammar(format, expected_grammar) check_stag_with_instance(format, instance, is_accepted) test_strings_is_accepted_excluded_any_text_within_sequence = [ ("HelloABC", True), ("WorldABC", True), ("NoExclusionHere", False), ("JustSomeText", False), ("ABC", True), ("SomeTextBeforeABC", True), ] @pytest.mark.parametrize( "instance, is_accepted", test_strings_is_accepted_excluded_any_text_within_sequence ) def test_excluded_any_text_within_sequence(instance: str, is_accepted: bool): format = { "type": "sequence", "elements": [ {"type": "any_text", "excludes": ["ABC"]}, {"type": "const_string", "value": "ABC"}, ], } expected_grammar = r"""any_text ::= TagDispatch( loop_after_dispatch=false, excludes=("ABC") ) const_string ::= (("ABC")) sequence ::= ((any_text const_string)) root ::= ((sequence)) """ check_stag_with_grammar(format, expected_grammar) check_stag_with_instance(format, instance, is_accepted) test_strings_is_accepted_excluded_triggered_tags_without_end = [ ("1ABC", False), ("11ABC", True), ("1HelloWorld", False), ("1ABC123", False), ("2ABC", True), ] @pytest.mark.parametrize( "instance, is_accepted", test_strings_is_accepted_excluded_triggered_tags_without_end ) def test_excludes_triggered_tags_without_end(instance: str, is_accepted: bool): stag = { "type": "sequence", "elements": [ { "type": "triggered_tags", "triggers": ["1"], "tags": [{"begin": "1", "content": {"type": "any_text"}, "end": ["1"]}], "excludes": ["ABC"], }, {"type": "const_string", "value": "ABC"}, ], } expected_grammar = r"""any_text ::= TagDispatch( loop_after_dispatch=false, excludes=("1") ) triggered_tags_group ::= (("" any_text "1")) triggered_tags ::= TagDispatch( ("1", triggered_tags_group), loop_after_dispatch=true, excludes=("ABC") ) const_string ::= (("ABC")) sequence ::= ((triggered_tags const_string)) root ::= ((sequence)) """ check_stag_with_grammar(stag, expected_grammar) check_stag_with_instance(stag, instance, is_accepted) # ==================== XML const/enum/anyOf string value tests ==================== def _make_xml_property_format(prop_schema, style="qwen_xml"): return { "type": "json_schema", "json_schema": {"type": "object", "properties": {"v": prop_schema}, "required": ["v"]}, "style": style, } xml_const_enum_instances = [ # String const: unquoted (_make_xml_property_format({"const": "hello"}), "hello", True), (_make_xml_property_format({"const": "hello"}), '"hello"', False), # Integer const: unchanged (_make_xml_property_format({"const": 42}), "42", True), (_make_xml_property_format({"const": 42}), "43", False), # Boolean const (_make_xml_property_format({"const": True}), "true", True), # Null const (_make_xml_property_format({"const": None}), "null", True), (_make_xml_property_format({"const": '"\\'}), '"\\', True), # String enum: unquoted (_make_xml_property_format({"enum": ["red", "green"]}), "red", True), ( _make_xml_property_format({"enum": ["red", "green"]}), '"red"', False, ), (_make_xml_property_format({"enum": ["red", "green"]}), "blue", False), # Mixed enum: string unquoted, integer raw ( _make_xml_property_format({"enum": ["hello", 42, '"\\']}), "hello", True, ), ( _make_xml_property_format({"enum": ["hello", 42, '"\\']}), "42", True, ), ( _make_xml_property_format({"enum": ["hello", 42, '"\\']}), '"\\', True, ), # anyOf with string const branches ( _make_xml_property_format({"anyOf": [{"const": "a"}, {"const": "b"}]}), "a", True, ), ( _make_xml_property_format({"anyOf": [{"const": "a"}, {"const": "b"}]}), "c", False, ), # anyOf with string + integer branches ( _make_xml_property_format({"anyOf": [{"type": "string"}, {"type": "integer"}]}), "hello world", True, ), ( _make_xml_property_format({"anyOf": [{"type": "string"}, {"type": "integer"}]}), "123", True, ), ] @pytest.mark.parametrize("stag_format, instance, is_accepted", xml_const_enum_instances) def test_xml_const_enum_values(stag_format: Dict[str, Any], instance: str, is_accepted: bool): check_stag_with_instance(stag_format, instance, is_accepted) # ==================== Token-level Format Tests ==================== # ---------- TokenFormat Tests ---------- def test_token_format_basic(): check_stag_with_grammar( {"type": "token", "token": 42}, r"""token ::= ((Token(42))) root ::= ((token)) """, ) def test_token_format_in_tag_begin_end(): check_stag_with_grammar( { "type": "tag", "begin": {"type": "token", "token": 10}, "content": {"type": "const_string", "value": "X"}, "end": {"type": "token", "token": 20}, }, r"""const_string ::= (("X")) tag ::= ((Token(10) const_string Token(20))) root ::= ((tag)) """, ) def test_token_format_in_tag_begin_string_end(): check_stag_with_grammar( { "type": "tag", "begin": {"type": "token", "token": 10}, "content": {"type": "const_string", "value": "Y"}, "end": "", }, r"""const_string ::= (("Y")) tag ::= ((Token(10) const_string "")) root ::= ((tag)) """, ) # ---------- ExcludeTokenFormat Tests ---------- def test_exclude_token_format_no_excludes(): check_stag_with_grammar( {"type": "exclude_token"}, r"""exclude_token ::= ((ExcludeToken())) root ::= ((exclude_token)) """, ) def test_exclude_token_format_with_excludes(): check_stag_with_grammar( {"type": "exclude_token", "exclude_tokens": [5, 10]}, r"""exclude_token ::= ((ExcludeToken(5, 10))) root ::= ((exclude_token)) """, ) def test_exclude_token_detects_end_from_parent_tag(): """ExcludeTokenFormat inside a tag with token end should auto-detect end token IDs.""" check_stag_with_grammar( { "type": "tag", "begin": {"type": "token", "token": 1}, "content": {"type": "exclude_token", "exclude_tokens": [5]}, "end": {"type": "token", "token": 99}, }, r"""exclude_token ::= ((ExcludeToken(5, 99))) tag ::= ((Token(1) exclude_token Token(99))) root ::= ((tag)) """, ) # ---------- AnyTokensFormat Tests ---------- def test_any_tokens_format_no_excludes(): check_stag_with_grammar( {"type": "any_tokens"}, r"""any_tokens_inner ::= ((ExcludeToken())) any_tokens ::= ("" | (any_tokens_inner any_tokens)) root ::= ((any_tokens)) """, ) def test_any_tokens_format_with_excludes(): check_stag_with_grammar( {"type": "any_tokens", "exclude_tokens": [5, 10]}, r"""any_tokens_inner ::= ((ExcludeToken(5, 10))) any_tokens ::= ("" | (any_tokens_inner any_tokens)) root ::= ((any_tokens)) """, ) def test_any_tokens_detects_end_from_parent_tag(): """AnyTokensFormat inside a tag with token end should auto-detect end token IDs.""" check_stag_with_grammar( { "type": "tag", "begin": {"type": "token", "token": 1}, "content": {"type": "any_tokens", "exclude_tokens": [5]}, "end": {"type": "token", "token": 99}, }, r"""any_tokens_inner ::= ((ExcludeToken(5, 99))) any_tokens ::= ("" | (any_tokens_inner any_tokens)) tag ::= ((Token(1) any_tokens Token(99))) root ::= ((tag)) """, ) # ---------- TokenTriggeredTagsFormat Tests ---------- def test_token_triggered_tags_stop_after_first(): check_stag_with_grammar( { "type": "token_triggered_tags", "trigger_tokens": [10, 20], "tags": [ { "type": "tag", "begin": {"type": "token", "token": 10}, "content": {"type": "const_string", "value": "A"}, "end": {"type": "token", "token": 99}, }, { "type": "tag", "begin": {"type": "token", "token": 20}, "content": {"type": "const_string", "value": "B"}, "end": {"type": "token", "token": 99}, }, ], "stop_after_first": True, }, r"""const_string ::= (("A")) const_string_1 ::= (("B")) token_triggered_tags_group ::= ((const_string Token(99))) token_triggered_tags_group_1 ::= ((const_string_1 Token(99))) token_triggered_tags ::= ((token_triggered_tags_1)) root ::= ((token_triggered_tags)) token_triggered_tags_1 ::= TokenTagDispatch( (10, token_triggered_tags_group), (20, token_triggered_tags_group_1), loop_after_dispatch=false, excludes=() ) """, ) def test_token_triggered_tags_at_least_one_stop_after_first(): check_stag_with_grammar( { "type": "token_triggered_tags", "trigger_tokens": [10], "tags": [ { "type": "tag", "begin": {"type": "token", "token": 10}, "content": {"type": "const_string", "value": "A"}, "end": {"type": "token", "token": 99}, } ], "at_least_one": True, "stop_after_first": True, }, r"""const_string ::= (("A")) token_triggered_tags ::= ((Token(10) const_string Token(99))) root ::= ((token_triggered_tags)) """, ) def test_token_triggered_tags_with_excludes(): check_stag_with_grammar( { "type": "token_triggered_tags", "trigger_tokens": [10], "tags": [ { "type": "tag", "begin": {"type": "token", "token": 10}, "content": {"type": "const_string", "value": "C"}, "end": {"type": "token", "token": 99}, } ], "exclude_tokens": [50], "stop_after_first": True, }, r"""const_string ::= (("C")) token_triggered_tags_group ::= ((const_string Token(99))) token_triggered_tags ::= ((token_triggered_tags_1)) root ::= ((token_triggered_tags)) token_triggered_tags_1 ::= TokenTagDispatch( (10, token_triggered_tags_group), loop_after_dispatch=false, excludes=(50) ) """, ) def test_token_triggered_tags_looping(): check_stag_with_grammar( { "type": "token_triggered_tags", "trigger_tokens": [10], "tags": [ { "type": "tag", "begin": {"type": "token", "token": 10}, "content": {"type": "const_string", "value": "D"}, "end": {"type": "token", "token": 99}, } ], }, r"""const_string ::= (("D")) token_triggered_tags_group ::= ((const_string Token(99))) token_triggered_tags ::= ((token_triggered_tags_1)) root ::= ((token_triggered_tags)) token_triggered_tags_1 ::= TokenTagDispatch( (10, token_triggered_tags_group), loop_after_dispatch=true, excludes=() ) """, ) def test_token_triggered_tags_detects_end_from_parent(): """TokenTriggeredTagsFormat inside a tag should auto-detect end token IDs.""" check_stag_with_grammar( { "type": "tag", "begin": {"type": "token", "token": 1}, "content": { "type": "token_triggered_tags", "trigger_tokens": [10], "tags": [ { "type": "tag", "begin": {"type": "token", "token": 10}, "content": {"type": "const_string", "value": "E"}, "end": {"type": "token", "token": 99}, } ], "stop_after_first": True, }, "end": {"type": "token", "token": 88}, }, r"""const_string ::= (("E")) token_triggered_tags_group ::= ((const_string Token(99))) token_triggered_tags ::= ((token_triggered_tags_1)) tag ::= ((Token(1) token_triggered_tags Token(88))) root ::= ((tag)) token_triggered_tags_1 ::= TokenTagDispatch( (10, token_triggered_tags_group), loop_after_dispatch=false, excludes=(88) ) """, ) # ---------- Token Format Parsing Error Tests ---------- def test_token_format_missing_token_field(): stag = {"type": "structural_tag", "format": {"type": "token"}} with pytest.raises(Exception, match="Invalid structural tag error"): xgr.Grammar.from_structural_tag(stag) def test_exclude_token_format_invalid_exclude_tokens_type(): stag = {"type": "structural_tag", "format": {"type": "exclude_token", "exclude_tokens": "bad"}} with pytest.raises(Exception, match="Invalid structural tag error"): xgr.Grammar.from_structural_tag(stag) def test_any_tokens_format_invalid_exclude_type(): stag = {"type": "structural_tag", "format": {"type": "any_tokens", "exclude_tokens": "bad"}} with pytest.raises(Exception, match="Invalid structural tag error"): xgr.Grammar.from_structural_tag(stag) def test_token_triggered_tags_missing_triggers(): stag = {"type": "structural_tag", "format": {"type": "token_triggered_tags", "tags": []}} with pytest.raises(Exception, match="Invalid structural tag error"): xgr.Grammar.from_structural_tag(stag) def test_token_triggered_tags_missing_tags(): stag = { "type": "structural_tag", "format": {"type": "token_triggered_tags", "trigger_tokens": [1]}, } with pytest.raises(Exception, match="Invalid structural tag error"): xgr.Grammar.from_structural_tag(stag) def test_token_string_requires_tokenizer(): """String tokens without tokenizer should error.""" stag = {"type": "structural_tag", "format": {"type": "token", "token": "<|special|>"}} with pytest.raises(Exception, match="Invalid structural tag error"): xgr.Grammar.from_structural_tag(stag) def test_triggered_tags_rejects_token_begin(): stag = { "type": "structural_tag", "format": { "type": "triggered_tags", "triggers": [""], "tags": [ { "type": "tag", "begin": {"type": "token", "token": 10}, "content": {"type": "const_string", "value": "X"}, "end": "", } ], }, } with pytest.raises(Exception, match="string begin"): xgr.Grammar.from_structural_tag(stag) def test_token_triggered_tags_rejects_string_begin(): stag = { "type": "structural_tag", "format": { "type": "token_triggered_tags", "trigger_tokens": [10], "tags": [ { "type": "tag", "begin": "", "content": {"type": "const_string", "value": "X"}, "end": {"type": "token", "token": 99}, } ], }, } with pytest.raises(Exception, match="token format begin"): xgr.Grammar.from_structural_tag(stag) # ---------- DispatchFormat ---------- tag_dispatch_format_stag = { "type": "dispatch", "rules": [ ["tag1", {"type": "const_string", "value": "abcd"}], ["tag2", {"type": "const_string", "value": "efg"}], ], "loop": True, } tag_dispatch_format_expected_grammar = "" tag_dispatch_format_instance_accepted = [ ("tag1abcd", True), ("tag1abcdtag2efg", True), ("tag1abcdqqqqtag2efg", True), ] tag_dispatch_format_instance_rejected = [ ("tag1abc", False), ("tag1abce", False), ("ttag1abd", False), ] @pytest.mark.parametrize( "instance, is_accepted", tag_dispatch_format_instance_accepted + tag_dispatch_format_instance_rejected, ) def test_tag_dispatch_format_simple(instance: str, is_accepted: bool): """DispatchFormat: positive/negative instances (cf. test_grammar_matcher_macro.test_simple).""" if tag_dispatch_format_expected_grammar: check_stag_with_grammar(tag_dispatch_format_stag, tag_dispatch_format_expected_grammar) check_stag_with_instance(tag_dispatch_format_stag, instance, is_accepted) tag_dispatch_format_no_loop_stag = { "type": "dispatch", "rules": [ ["tag1", {"type": "const_string", "value": "abcd"}], ["tag2", {"type": "const_string", "value": "efg"}], ], "loop": False, } tag_dispatch_format_no_loop_expected_grammar = r"""const_string ::= (("abcd")) const_string_1 ::= (("efg")) tag_dispatch ::= TagDispatch( ("tag1", const_string), ("tag2", const_string_1), loop_after_dispatch=false, excludes=() ) root ::= ((tag_dispatch)) """ tag_dispatch_format_no_loop_instance_accepted = [("tag1abcd", True), ("tag2efg", True)] tag_dispatch_format_no_loop_instance_rejected = [ ("tag1abcdtag2efg", False), ("tag2efgtag1abcd", False), ] @pytest.mark.parametrize( "instance, is_accepted", tag_dispatch_format_no_loop_instance_accepted + tag_dispatch_format_no_loop_instance_rejected, ) def test_tag_dispatch_format_no_loop(instance: str, is_accepted: bool): """DispatchFormat with loop=false (cf. test_grammar_matcher_macro.test_no_loop_after_dispatch).""" check_stag_with_grammar( tag_dispatch_format_no_loop_stag, tag_dispatch_format_no_loop_expected_grammar ) check_stag_with_instance(tag_dispatch_format_no_loop_stag, instance, is_accepted) tag_dispatch_format_with_excludes_stag = { "type": "dispatch", "rules": [ ["tag1", {"type": "const_string", "value": "abcd"}], ["tag2", {"type": "const_string", "value": "efg"}], ], "loop": True, "excludes": ["tag3", "ll"], } tag_dispatch_format_with_excludes_expected_grammar = r"""const_string ::= (("abcd")) const_string_1 ::= (("efg")) tag_dispatch ::= TagDispatch( ("tag1", const_string), ("tag2", const_string_1), loop_after_dispatch=true, excludes=("tag3", "ll") ) root ::= ((tag_dispatch)) """ tag_dispatch_format_with_excludes_instance_accepted = [ ("tag1abcd123", True), ("tag1abcdqqqtag2efg12W3", True), ] tag_dispatch_format_with_excludes_instance_rejected = [ ("tag1abcdll", False), ("tag1abcdlltag3", False), ] @pytest.mark.parametrize( "instance, is_accepted", tag_dispatch_format_with_excludes_instance_accepted + tag_dispatch_format_with_excludes_instance_rejected, ) def test_tag_dispatch_format_with_excludes(instance: str, is_accepted: bool): """DispatchFormat with excludes (cf. test_grammar_matcher_macro.test_stop_str).""" check_stag_with_grammar( tag_dispatch_format_with_excludes_stag, tag_dispatch_format_with_excludes_expected_grammar ) check_stag_with_instance(tag_dispatch_format_with_excludes_stag, instance, is_accepted) # ---------- TokenDispatchFormat ---------- def test_token_tag_dispatch_format_simple(): """TokenDispatchFormat: two trigger tokens, each with const_string content.""" stag_format = { "type": "token_dispatch", "rules": [ [10, {"type": "const_string", "value": "A"}], [20, {"type": "const_string", "value": "B"}], ], "loop": False, } expected_grammar = r"""const_string ::= (("A")) const_string_1 ::= (("B")) token_tag_dispatch ::= ((token_tag_dispatch_1)) root ::= ((token_tag_dispatch)) token_tag_dispatch_1 ::= TokenTagDispatch( (10, const_string), (20, const_string_1), loop_after_dispatch=false, excludes=() ) """ check_stag_with_grammar(stag_format, expected_grammar) def test_token_tag_dispatch_format_with_excludes(): """TokenDispatchFormat with exclude_tokens.""" stag_format = { "type": "token_dispatch", "rules": [[10, {"type": "const_string", "value": "C"}]], "loop": False, "exclude_tokens": [50], } expected_grammar = r"""const_string ::= (("C")) token_tag_dispatch ::= ((token_tag_dispatch_1)) root ::= ((token_tag_dispatch)) token_tag_dispatch_1 ::= TokenTagDispatch( (10, const_string), loop_after_dispatch=false, excludes=(50) ) """ check_stag_with_grammar(stag_format, expected_grammar) def test_token_tag_dispatch_format_looping(): """TokenDispatchFormat with loop=true.""" stag_format = { "type": "token_dispatch", "rules": [[10, {"type": "const_string", "value": "D"}]], "loop": True, } expected_grammar = r"""const_string ::= (("D")) token_tag_dispatch ::= ((token_tag_dispatch_1)) root ::= ((token_tag_dispatch)) token_tag_dispatch_1 ::= TokenTagDispatch( (10, const_string), loop_after_dispatch=true, excludes=() ) """ check_stag_with_grammar(stag_format, expected_grammar) def test_token_format_rejects_float(): stag = {"type": "structural_tag", "format": {"type": "token", "token": 3.5}} with pytest.raises(Exception, match="must be an integer"): xgr.Grammar.from_structural_tag(stag) def test_token_tag_dispatch_need_tokenizer_info(): stag = { "type": "structural_tag", "format": { "type": "token_dispatch", "rules": [["<|tag|>", {"type": "const_string", "value": "abcd"}]], }, } with pytest.raises(Exception, match="Invalid structural tag error"): xgr.Grammar.from_structural_tag(stag) # ---------- JSONSchemaFormat with any_order ---------- any_order_schema = { "type": "object", "properties": {"a": {"type": "integer"}, "b": {"type": "string"}, "c": {"type": "boolean"}}, "required": ["a", "b"], "additionalProperties": False, } # The full acceptance matrix is covered at the converter level (test_any_order_acceptance); here we # just confirm any_order is parsed from the structural tag and threaded through to the grammar. any_order_json_instance_is_accepted = [ ('{"c": true, "a": 1, "b": "x"}', True), # reordered/interleaved -> any_order is applied ('{"a": 1, "b": "x", "c": true, "c": false}', True), # other entries are not count-limited ('{"a": 1}', False), # fewer entries than #required ('{"a": 1, "b": "x", "d": 5}', False), # additionalProperties false ] @pytest.mark.parametrize("instance, is_accepted", any_order_json_instance_is_accepted) def test_json_schema_format_any_order(instance: str, is_accepted: bool): stag_format = {"type": "json_schema", "json_schema": any_order_schema, "any_order": True} check_stag_with_instance(stag_format, instance, is_accepted) def test_json_schema_format_any_order_default_is_ordered(): # Without any_order (the default), the JSON schema keeps the fixed declared order. stag_format = {"type": "json_schema", "json_schema": any_order_schema} check_stag_with_instance(stag_format, '{"a": 1, "b": "x"}', True) check_stag_with_instance(stag_format, '{"b": "x", "a": 1}', False) any_order_additional_schema = { "type": "object", "properties": {"a": {"type": "integer"}, "b": {"type": "string"}}, "required": ["a"], "additionalProperties": True, } any_order_additional_instance_is_accepted = [ ('{"a": 1}', True), ('{"a": 1, "b": "x"}', True), ('{"a": 1, "z": 5, "y": "q", "w": true}', True), # additional keys, unbounded, any order ('{"z": 5, "a": 1}', True), # additional before required (interleaved freely) ("{}", False), # 0 entries < n = max(minProperties, #required) = 1 ] @pytest.mark.parametrize("instance, is_accepted", any_order_additional_instance_is_accepted) def test_json_schema_format_any_order_additional_properties(instance: str, is_accepted: bool): stag_format = { "type": "json_schema", "json_schema": any_order_additional_schema, "any_order": True, } check_stag_with_instance(stag_format, instance, is_accepted) any_order_xml_schema = { "type": "object", "properties": {"a": {"type": "integer"}, "b": {"type": "string"}, "c": {"type": "boolean"}}, "required": ["a", "b"], "additionalProperties": False, } any_order_xml_instance_is_accepted = [ ("1hello", True), # declared order ("hello1", True), # reordered required # optional field after the required group, exercised through the XML tail ("1hellotrue", True), ("1", False), # 1 entry < n = #required = 2 # rejected because is not a declared key (additionalProperties: false), via XML ( "1hello" "truefalse", False, ), ] @pytest.mark.parametrize("instance, is_accepted", any_order_xml_instance_is_accepted) def test_json_schema_format_any_order_qwen_xml(instance: str, is_accepted: bool): stag_format = { "type": "json_schema", "json_schema": any_order_xml_schema, "style": "qwen_xml", "any_order": True, } check_stag_with_instance(stag_format, instance, is_accepted) def test_json_schema_format_any_order_via_pydantic(): # any_order also works when constructing the structural tag through the pydantic models, and # round-trips through serialization (JSONSchemaFormat.ToJSON / ParseJSONSchemaFormat). from xgrammar.structural_tag import JSONSchemaFormat st_any = StructuralTag(format=JSONSchemaFormat(json_schema=any_order_schema, any_order=True)) check_stag_with_instance(st_any, '{"b": "x", "a": 1}', True) st_ordered = StructuralTag( format=JSONSchemaFormat(json_schema=any_order_schema, any_order=False) ) check_stag_with_instance(st_ordered, '{"b": "x", "a": 1}', False) if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.2.3/tests/python/test_token_bitmask_operations.py000066400000000000000000000366161521764210300245400ustar00rootroot00000000000000"""This test uses the optimized JSON grammar provided by the grammar library.""" import sys import time from typing import Callable, List, Optional, Tuple import pytest import torch try: import mlx_lm # noqa: F401 MLX_AVAILABLE = True except ModuleNotFoundError: MLX_AVAILABLE = False import xgrammar as xgr from xgrammar.testing import ( _get_masked_tokens_from_bitmask, _is_single_token_bitmask, bitmask_to_bool_mask, bool_mask_to_bitmask, ) _is_cuda_available = torch.cuda.is_available() _is_mlx_metal_available = torch.backends.mps.is_available() and MLX_AVAILABLE def test_allocate_reset_token_bitmask(): batch_size = 10 vocab_size = 128005 bitmask = xgr.allocate_token_bitmask(batch_size, vocab_size) assert bitmask.shape == (batch_size, (vocab_size + 31) // 32) assert bitmask.device.type == "cpu" assert (bitmask == 0xFFFFFFFF).all() bitmask.fill_(0) xgr.reset_token_bitmask(bitmask) assert (bitmask == 0xFFFFFFFF).all() token_mask_sizes = (1024, 32000, 32001, 32011) @pytest.mark.parametrize("token_mask_size", token_mask_sizes) @pytest.mark.parametrize("index", (0, 1)) def test_get_masked_tokens_from_bitmask(token_mask_size: int, index: int): bool_mask = torch.randint(0, 2, (2, token_mask_size), dtype=torch.bool) bitmask = bool_mask_to_bitmask(bool_mask) expected = torch.where(~bool_mask[index])[0].tolist() assert _get_masked_tokens_from_bitmask(bitmask, token_mask_size, index) == expected def test_is_single_token_bitmask(): batch = 2 batch_index = 1 vocab_size = 1024 token_id = 100 bool_mask = torch.zeros(batch, vocab_size, dtype=torch.bool) bitmask = bool_mask_to_bitmask(bool_mask) assert _is_single_token_bitmask(bitmask, vocab_size, batch_index) == (False, -1) bool_mask[batch_index, token_id] = True bitmask = bool_mask_to_bitmask(bool_mask) assert _is_single_token_bitmask(bitmask, vocab_size, batch_index) == (True, token_id) bool_mask[batch_index, token_id + 1] = True bitmask = bool_mask_to_bitmask(bool_mask) assert _is_single_token_bitmask(bitmask, vocab_size, batch_index) == (False, -1) @pytest.mark.parametrize("device", ("cpu", "cuda")) def test_apply_token_bitmask_inplace(device: str): if device == "cuda" and not _is_cuda_available: pytest.skip(reason="CUDA is not installed") neginf = float("-inf") bool_mask = torch.tensor([0, 1, 0, 1, 0, 1, 0, 1, 0, 1], dtype=torch.bool, device=device) bitmask = torch.tensor([0b1010101010], dtype=torch.int32, device=device) logits = torch.tensor( [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0], dtype=torch.float32, device=device ) expected = torch.where(bool_mask, logits, neginf) xgr.apply_token_bitmask_inplace(logits, bitmask) torch.testing.assert_close(logits, expected) @pytest.mark.parametrize("device", ("cpu", "cuda")) def test_apply_token_bitmask_inplace_shape_stride_mismatch(device: str): if device == "cuda" and not _is_cuda_available: pytest.skip(reason="CUDA is not installed") col = 100 compacted_col = (col + 31) // 32 neginf = float("-inf") # Mask even positions (0-indexed) in the first row, and # mask odd positions in the second row. bool_mask = torch.tensor( [[i % 2 == 0 for i in range(col)], [i % 2 == 1 for i in range(col)]], dtype=torch.bool, device=device, ) # In int32 binary representation, # 0x55555555 = 1431655765 # 0xAAAAAAAA = -1431655766 bitmask = torch.tensor( [[1431655765] * compacted_col, [-1431655766] * compacted_col], dtype=torch.int32, device=device, ) master_logits = torch.tensor( [[i + 0.1 for i in range(col + 1)], [i + 0.2 for i in range(col + 1)]], dtype=torch.float32, device=device, ) logits = master_logits[:, :col] # Ensure the test environment setup is accurate (i.e. shape[-1] != stride[0]) assert logits.size() == (2, col) assert logits.stride() == (col + 1, 1) expected = torch.where(bool_mask, logits, neginf) xgr.apply_token_bitmask_inplace(logits, bitmask) torch.testing.assert_close(logits, expected) def get_apply_token_bitmask_kernel(impl: str) -> Callable: if impl == "cpu": from xgrammar.kernels.apply_token_bitmask_inplace_cpu import apply_token_bitmask_inplace_cpu return apply_token_bitmask_inplace_cpu elif impl == "cuda": from xgrammar.kernels.apply_token_bitmask_inplace_cuda import ( apply_token_bitmask_inplace_cuda, ) return apply_token_bitmask_inplace_cuda elif impl == "triton": from xgrammar.kernels.apply_token_bitmask_inplace_triton import ( apply_token_bitmask_inplace_triton, ) return apply_token_bitmask_inplace_triton elif impl == "metal": from xgrammar.kernels.apply_token_bitmask_mlx import apply_token_bitmask_mlx return apply_token_bitmask_mlx elif impl == "torch_compile": from xgrammar.kernels.apply_token_bitmask_inplace_torch_compile import ( apply_token_bitmask_inplace_torch_compile, ) return apply_token_bitmask_inplace_torch_compile else: raise ValueError(f"Invalid implementation: {impl}") @pytest.mark.parametrize("impl", ("cpu", "cuda", "triton", "metal", "torch_compile")) def test_apply_token_bitmask_inplace_kernel(impl: str, num_parallel_threads: int): if impl in ["cuda", "triton", "torch_compile"] and not _is_cuda_available: pytest.skip(reason="CUDA is not installed") elif impl == "metal" and not _is_mlx_metal_available: pytest.skip(reason="MLX is not installed") elif impl == "metal" and num_parallel_threads > 1: pytest.skip(reason="MLX crashes under multithreading") kernel = get_apply_token_bitmask_kernel(impl) neginf = float("-inf") bool_mask = torch.tensor([0, 1, 0, 1, 0, 1, 0, 1, 0, 1], dtype=torch.bool) logits = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0], dtype=torch.float32) expected = torch.where(bool_mask, logits, neginf) if impl in ["cuda", "triton", "torch_compile"]: logits_gpu = logits.to("cuda") bitmask = torch.tensor([0b1010101010], dtype=torch.int32).to("cuda") kernel(logits_gpu, bitmask) torch.cuda.synchronize() torch.testing.assert_close(logits_gpu, expected.to("cuda")) elif impl == "metal": # Import MLX only when needed for the Metal test import mlx.core as mx bitmask = mx.array([0b1010101010], dtype=mx.int32) logits = mx.array(logits.numpy()) result = kernel(bitmask, logits, vocab_size=10) expected = mx.array(expected.numpy()) assert mx.allclose(result, expected) else: assert impl == "cpu" bitmask = torch.tensor([0b1010101010], dtype=torch.int32) kernel(logits, bitmask) torch.testing.assert_close(logits, expected) batch_size__vocab_size__masked_cnt__stride__logits_dtype = [ (1, 128000, 1024, 1, "float32"), (1, 128000, 120000, 1, "float32"), (1, 128001, 120000, 1, "float32"), (1, 128010, 120000, 1, "float32"), (64, 128000, 1024, 1, "float32"), (64, 128000, 120000, 1, "float32"), (64, 128000, 1024, 4, "float32"), (64, 128000, 120000, 4, "float32"), (64, 128001, 120000, 1, "float32"), (64, 128010, 120000, 1, "float32"), (64, 128000, 1024, 1, "float16"), (64, 128000, 1024, 1, "bfloat16"), ] @pytest.mark.parametrize( "batch_size, vocab_size, masked_cnt, stride, logits_dtype", batch_size__vocab_size__masked_cnt__stride__logits_dtype, ) @pytest.mark.parametrize("impl", ("cpu", "cuda", "triton", "torch_compile")) def test_apply_token_bitmask_inplace_kernel_large( batch_size: int, vocab_size: int, masked_cnt: int, stride: int, logits_dtype: str, impl: str ): if impl in ["cuda", "triton", "torch_compile"] and not _is_cuda_available: pytest.skip(reason="CUDA is not installed") kernel = get_apply_token_bitmask_kernel(impl) logits_dtype = getattr(torch, logits_dtype) logits = torch.randn(batch_size, vocab_size, dtype=logits_dtype) if masked_cnt >= vocab_size: bool_mask = torch.zeros(batch_size, vocab_size, dtype=torch.bool) else: bool_mask = torch.ones(batch_size, vocab_size, dtype=torch.bool) if masked_cnt > 0: masked_positions = torch.stack( [torch.randperm(vocab_size)[:masked_cnt] for _ in range(batch_size)] ) bool_mask.scatter_(1, masked_positions, False) assert (bool_mask.sum(dim=-1) + masked_cnt == vocab_size).all().item() bitmask = bool_mask_to_bitmask(bool_mask) batch_indices = torch.arange(0, batch_size, stride, dtype=torch.int32) logits_expected = logits.clone() logits_expected[batch_indices] = torch.masked_fill( logits_expected[batch_indices], ~bool_mask[batch_indices], float("-inf") ) bitmask = bool_mask_to_bitmask(bool_mask) if impl in ["cuda", "triton", "torch_compile"]: logits_gpu = logits.to("cuda") bitmask_gpu = bitmask.to("cuda") indices = batch_indices.to("cuda") if stride != 1 else None f = lambda: kernel(logits_gpu, bitmask_gpu, indices=indices) torch.cuda.synchronize() f() torch.cuda.synchronize() torch.testing.assert_close(logits_gpu, logits_expected.to("cuda")) try: from triton.testing import do_bench exec_time = do_bench(f, warmup=100, rep=1000) exec_time *= 1e3 except ImportError: pytest.skip(reason="Triton is not installed") else: assert impl == "cpu" indices = batch_indices.tolist() if stride != 1 else None time_start = time.monotonic_ns() kernel(logits, bitmask, indices=indices) time_end = time.monotonic_ns() exec_time = (time_end - time_start) / 1e3 torch.testing.assert_close(logits, logits_expected) print( f"Batch: {batch_size:2} | Vocab: {vocab_size:6} | Masked: {masked_cnt:6} | " f"Stride: {stride:1} | DType: {str(logits_dtype):15} | Impl: {impl:6} | " f"Execution time (μs): {exec_time:.4f}" ) logits_shape__bitmask_shape__vocab_size = [ # logits is larger ((2, 130), (2, 4), None), # bitmask is larger ((2, 120), (2, 4), None), # vocab size is specified ((2, 130), (2, 4), 120), ] @pytest.mark.parametrize( "logits_shape, bitmask_shape, vocab_size", logits_shape__bitmask_shape__vocab_size ) @pytest.mark.parametrize("impl", ("cpu", "triton", "torch_compile")) def test_apply_token_bitmask_inplace_vocab_size( logits_shape: Tuple[int, int], bitmask_shape: Tuple[int, int], vocab_size: Optional[int], impl: str, ): if impl in ["triton", "torch_compile"] and not _is_cuda_available: pytest.skip(reason="CUDA is not installed") kernel = get_apply_token_bitmask_kernel(impl) logits_dtype = torch.float32 logits = torch.ones(logits_shape, dtype=logits_dtype) bitmask = torch.zeros(bitmask_shape, dtype=torch.int32) vocab_size = min(logits_shape[1], bitmask_shape[1] * 32) if vocab_size is None else vocab_size logits_expected = logits.clone() logits_expected[..., :vocab_size] = float("-inf") if impl in ["triton", "torch_compile"]: logits_gpu = logits.to("cuda") bitmask_gpu = bitmask.to("cuda") kernel(logits_gpu, bitmask_gpu, vocab_size=vocab_size) torch.testing.assert_close(logits_gpu, logits_expected.to("cuda")) else: assert impl == "cpu" kernel(logits, bitmask, vocab_size=vocab_size) torch.testing.assert_close(logits, logits_expected) logits_batch_size__bitmask_batch_size__vocab_size__indices = [ (3, 3, 128, [0, 1]), (2, 3, 128, [0]), (3, 2, 130, [0]), ] @pytest.mark.parametrize( "logits_batch_size, bitmask_batch_size, vocab_size, indices", logits_batch_size__bitmask_batch_size__vocab_size__indices, ) @pytest.mark.parametrize("impl", ("cpu", "cuda", "triton", "torch_compile")) def test_apply_token_bitmask_inplace_indices( logits_batch_size: int, bitmask_batch_size: int, vocab_size: int, indices: List[int], impl: str ): if impl in ["cuda", "triton", "torch_compile"] and not _is_cuda_available: pytest.skip(reason="CUDA is not installed") kernel = get_apply_token_bitmask_kernel(impl) logits = torch.ones(logits_batch_size, vocab_size, dtype=torch.float32) bool_mask = torch.zeros(bitmask_batch_size, vocab_size, dtype=torch.bool) bitmask = bool_mask_to_bitmask(bool_mask) logits_expected = logits.clone() logits_expected[indices] = torch.masked_fill( logits_expected[indices], ~bool_mask[indices], float("-inf") ) if impl in ["cuda", "triton", "torch_compile"]: logits_gpu = logits.to("cuda") bitmask_gpu = bitmask.to("cuda") kernel(logits_gpu, bitmask_gpu, indices=indices) torch.testing.assert_close(logits_gpu, logits_expected.to("cuda")) else: assert impl == "cpu" kernel(logits, bitmask, indices=indices) torch.testing.assert_close(logits, logits_expected) def test_bitmask_to_boolmask(): # 0xFFFF0000, 0x0000FFFF bitmask = torch.tensor([[-65536, 65535]], dtype=torch.int32) expected = torch.tensor( [[False] * 16, [True] * 16, [True] * 16, [False] * 16], dtype=torch.bool ).reshape(1, -1) bool_mask = bitmask_to_bool_mask(bitmask) assert torch.equal(bool_mask, expected) bool_mask_50 = bitmask_to_bool_mask(bitmask, vocab_size=50) expected_50 = expected[:, :50] assert torch.equal(bool_mask_50, expected_50) batch__size__vocab__size = [ (4, 1000), (1, 1024), (16, 1024), # not a multiple of 16. (3, 817), ] @pytest.mark.parametrize("batch_size, vocab_size", batch__size__vocab__size) def test_bool_mask_bitmask_roundtrip(batch_size: int, vocab_size: int): bool_mask = torch.randint(0, 2, (batch_size, vocab_size), dtype=torch.bool) bitmask = bool_mask_to_bitmask(bool_mask) bool_mask_converted = bitmask_to_bool_mask(bitmask, vocab_size=vocab_size) assert torch.equal(bool_mask, bool_mask_converted) def test_apply_token_bitmask_inplace_cpu_bf16(): neginf = float("-inf") bool_mask = torch.tensor([0, 1, 0, 1, 0, 1, 0, 1, 0, 1], dtype=torch.bool, device="cpu") bitmask = torch.tensor([0b1010101010], dtype=torch.int32, device="cpu") logits = torch.tensor( [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0], dtype=torch.bfloat16, device="cpu" ) expected = torch.where(bool_mask, logits, neginf) xgr.apply_token_bitmask_inplace(logits, bitmask) torch.testing.assert_close(logits, expected) @pytest.mark.parametrize( "logits_batch_size, bitmask_batch_size, vocab_size, indices", logits_batch_size__bitmask_batch_size__vocab_size__indices, ) def test_apply_token_bitmask_inplace_indices_bf16( logits_batch_size: int, bitmask_batch_size: int, vocab_size: int, indices: List[int] ): logits = torch.ones(logits_batch_size, vocab_size, dtype=torch.bfloat16) bool_mask = torch.zeros(bitmask_batch_size, vocab_size, dtype=torch.bool) bitmask = bool_mask_to_bitmask(bool_mask) kernel = get_apply_token_bitmask_kernel("cpu") logits_expected = logits.clone() logits_expected[indices] = torch.masked_fill( logits_expected[indices], ~bool_mask[indices], float("-inf") ) kernel(logits, bitmask, indices=indices) torch.testing.assert_close(logits, logits_expected) if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.2.3/tests/python/test_token_edge.py000066400000000000000000001624061521764210300215440ustar00rootroot00000000000000"""Tests for Token() edge support in grammar parsing, matching, and bitmask generation.""" import sys import pytest import xgrammar as xgr from xgrammar.testing import ( _ebnf_to_grammar_no_normalization, _get_masked_tokens_from_bitmask, _get_matcher_from_grammar_and_tokenizer_info, ) # --- Parser / Printer roundtrip tests --- def test_parse_token_basic(): before = "root ::= Token(1, 2, 3)\n" expected = "root ::= ((Token(1, 2, 3)))\n" grammar = _ebnf_to_grammar_no_normalization(before) assert str(grammar) == expected def test_parse_token_single(): before = "root ::= Token(42)\n" expected = "root ::= ((Token(42)))\n" grammar = _ebnf_to_grammar_no_normalization(before) assert str(grammar) == expected def test_parse_token_sorted_deduped(): before = "root ::= Token(3, 1, 2, 1, 3)\n" expected = "root ::= ((Token(1, 2, 3)))\n" grammar = _ebnf_to_grammar_no_normalization(before) assert str(grammar) == expected def test_parse_token_in_sequence(): before = 'root ::= Token(1, 2) "hello"\n' expected = 'root ::= ((Token(1, 2) "hello"))\n' grammar = _ebnf_to_grammar_no_normalization(before) assert str(grammar) == expected def test_parse_token_in_alternation(): before = 'root ::= Token(1) | "hello"\n' expected = 'root ::= ((Token(1)) | ("hello"))\n' grammar = _ebnf_to_grammar_no_normalization(before) assert str(grammar) == expected def test_parse_exclude_token_basic(): before = "root ::= ExcludeToken(1, 2, 3)\n" expected = "root ::= ((ExcludeToken(1, 2, 3)))\n" grammar = _ebnf_to_grammar_no_normalization(before) assert str(grammar) == expected def test_parse_exclude_token_sorted_deduped(): before = "root ::= ExcludeToken(3, 1, 2, 1)\n" expected = "root ::= ((ExcludeToken(1, 2, 3)))\n" grammar = _ebnf_to_grammar_no_normalization(before) assert str(grammar) == expected # --- Matcher accept_token tests --- STOP_TOKEN_ID = 1 # "" in our test vocab def _make_matcher(vocab, grammar_str): """Create a matcher with a custom vocab and grammar.""" tokenizer_info = xgr.TokenizerInfo(vocab) grammar = xgr.Grammar.from_ebnf(grammar_str) return _get_matcher_from_grammar_and_tokenizer_info(grammar, tokenizer_info) def test_accept_token_basic(): """Token(2, 4) should accept token IDs 2 and 4 but reject others.""" vocab = ["", "", "aa", "bb", "cc", "dd"] # 0 1 2 3 4 5 matcher = _make_matcher(vocab, "root ::= Token(2, 4)\n") assert matcher.accept_token(2) assert matcher.accept_token(STOP_TOKEN_ID) assert matcher.is_terminated() def test_accept_token_reject(): """Tokens not in the Token() set should be rejected.""" vocab = ["", "", "aa", "bb", "cc", "dd"] matcher = _make_matcher(vocab, "root ::= Token(2, 4)\n") assert not matcher.accept_token(3) assert not matcher.accept_token(5) assert matcher.accept_token(4) assert matcher.accept_token(STOP_TOKEN_ID) assert matcher.is_terminated() def test_token_then_string(): """Token followed by string literal: Token(2) "bb" .""" vocab = ["", "", "aa", "bb", "cc"] matcher = _make_matcher(vocab, 'root ::= Token(2) "bb"\n') assert matcher.accept_token(2) # Token(2) = "aa" assert matcher.accept_token(3) # "bb" assert matcher.accept_token(STOP_TOKEN_ID) assert matcher.is_terminated() def test_token_or_string(): """Alternation: Token(2) | "bb" .""" vocab = ["", "", "aa", "bb", "cc"] # Accept via token path matcher = _make_matcher(vocab, 'root ::= Token(2) | "bb"\n') assert matcher.accept_token(2) assert matcher.accept_token(STOP_TOKEN_ID) assert matcher.is_terminated() # Accept via string path matcher2 = _make_matcher(vocab, 'root ::= Token(2) | "bb"\n') assert matcher2.accept_token(3) # "bb" assert matcher2.accept_token(STOP_TOKEN_ID) assert matcher2.is_terminated() # --- Bitmask tests --- def test_bitmask_token_only(): """FillNextTokenBitmask should allow only tokens in Token() set (and stop token).""" vocab = ["", "", "aa", "bb", "cc", "dd"] tokenizer_info = xgr.TokenizerInfo(vocab) grammar = xgr.Grammar.from_ebnf("root ::= Token(2, 4)\n") matcher = _get_matcher_from_grammar_and_tokenizer_info(grammar, tokenizer_info) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(token_bitmask) rejected = set(_get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size)) assert rejected == {0, 1, 3, 5} def test_bitmask_token_and_string(): """Bitmask for Token(2) | "bb" should allow token 2 and token whose text is "bb".""" vocab = ["", "", "aa", "bb", "cc"] tokenizer_info = xgr.TokenizerInfo(vocab) grammar = xgr.Grammar.from_ebnf('root ::= Token(2) | "bb"\n') matcher = _get_matcher_from_grammar_and_tokenizer_info(grammar, tokenizer_info) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(token_bitmask) rejected = set(_get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size)) assert rejected == {0, 1, 4} def test_bitmask_after_token(): """After accepting a Token, the bitmask should reflect the next expected tokens.""" vocab = ["", "", "aa", "bb", "cc"] tokenizer_info = xgr.TokenizerInfo(vocab) grammar = xgr.Grammar.from_ebnf('root ::= Token(2) "bb"\n') matcher = _get_matcher_from_grammar_and_tokenizer_info(grammar, tokenizer_info) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(token_bitmask) rejected = set(_get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size)) assert rejected == {0, 1, 3, 4} assert matcher.accept_token(2) matcher.fill_next_token_bitmask(token_bitmask) rejected2 = set(_get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size)) assert rejected2 == {0, 1, 2, 4} def test_token_multiple_choices(): """Token set with multiple IDs in alternation with other rules.""" vocab = ["", "", "x", "y", "z", "w"] tokenizer_info = xgr.TokenizerInfo(vocab) grammar = xgr.Grammar.from_ebnf('root ::= Token(2, 3, 4) | "w"\n') matcher = _get_matcher_from_grammar_and_tokenizer_info(grammar, tokenizer_info) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(token_bitmask) rejected = set(_get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size)) assert rejected == {0, 1} # --- 4.3 char-then-token sequence tests --- def test_char_then_token_sequence(): """String literal followed by Token: "A" Token(4, 5).""" vocab = ["", "", "A", "B", "hello", "world"] matcher = _make_matcher(vocab, 'root ::= "A" Token(4, 5)\n') assert matcher.accept_token(2) # "A" assert matcher.accept_token(4) # Token(4) = "hello" assert matcher.accept_token(STOP_TOKEN_ID) assert matcher.is_terminated() # --- TokenTagDispatch + excludes tests --- def _make_bitmask_helper(vocab, grammar_str): """Create matcher, tokenizer_info, and bitmask for a grammar.""" tokenizer_info = xgr.TokenizerInfo(vocab) grammar = xgr.Grammar.from_ebnf(grammar_str) matcher = _get_matcher_from_grammar_and_tokenizer_info(grammar, tokenizer_info) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) return matcher, tokenizer_info, token_bitmask def _get_accepted(matcher, token_bitmask, vocab_size): """Fill bitmask and return accepted token IDs.""" matcher.fill_next_token_bitmask(token_bitmask) rejected = _get_masked_tokens_from_bitmask(token_bitmask, vocab_size) return set(range(vocab_size)) - set(rejected) def test_token_tag_dispatch_exclude_no_triggers(): """ExcludeToken self-loop accepts all tokens except excluded ones.""" vocab = ["", "", "hello", "world", "blocked_1", "blocked_2"] grammar_str = """root ::= TokenTagDispatch( excludes=(4, 5) )""" matcher, ti, bitmask = _make_bitmask_helper(vocab, grammar_str) for _ in range(3): assert _get_accepted(matcher, bitmask, ti.vocab_size) == {0, 1, 2, 3} matcher.accept_token(2) def test_token_tag_dispatch_exclude_basic(): """ExcludeToken edge blocks excluded tokens.""" vocab = ["", "", "hello", "world", "bad"] grammar_str = """root ::= TokenTagDispatch( excludes=(4,) )""" matcher, ti, bitmask = _make_bitmask_helper(vocab, grammar_str) assert _get_accepted(matcher, bitmask, ti.vocab_size) == {0, 1, 2, 3} def test_token_tag_dispatch_reject_enforced_by_parser(): """accept_token must reject tokens excluded by kExcludeToken edge.""" vocab = ["", "", "hello", "world", "blocked"] grammar_str = """root ::= TokenTagDispatch( excludes=(4,) )""" ti = xgr.TokenizerInfo(vocab) grammar = xgr.Grammar.from_ebnf(grammar_str) matcher = _get_matcher_from_grammar_and_tokenizer_info(grammar, ti) assert not matcher.accept_token(4), "parser must reject excluded token" assert matcher.accept_token(2) # "hello" still accepted def test_token_tag_dispatch_trigger_and_exclude(): """TokenTagDispatch with trigger and exclude.""" vocab = ["", "", "A", "AB", "blocked"] grammar_str = """ rule1 ::= "done" root ::= TokenTagDispatch( (3, rule1), excludes=(4,) )""" matcher, ti, bitmask = _make_bitmask_helper(vocab, grammar_str) assert _get_accepted(matcher, bitmask, ti.vocab_size) == {0, 1, 2, 3} # --- TokenTagDispatch + trigger tests --- def test_token_tag_dispatch_trigger(): """Token trigger dispatches to a rule.""" vocab = ["", "", "hello", "trigger_tok", "content"] grammar_str = """ triggered_rule ::= Token(4) root ::= TokenTagDispatch( (3, triggered_rule) )""" matcher, ti, bitmask = _make_bitmask_helper(vocab, grammar_str) assert _get_accepted(matcher, bitmask, ti.vocab_size) == {0, 1, 2, 3, 4} assert matcher.accept_token(3) # dispatch trigger assert _get_accepted(matcher, bitmask, ti.vocab_size) == {4} def test_token_tag_dispatch_multiple_triggers(): """Multiple token triggers in TokenTagDispatch.""" vocab = ["", "", "A", "B", "", "content"] grammar_str = """ tool_body ::= Token(5) other_body ::= Token(5) root ::= TokenTagDispatch( (3, tool_body), (4, other_body) )""" matcher, ti, bitmask = _make_bitmask_helper(vocab, grammar_str) assert _get_accepted(matcher, bitmask, ti.vocab_size) == {0, 1, 2, 3, 4, 5} assert matcher.accept_token(3) # dispatch to tool_body assert _get_accepted(matcher, bitmask, ti.vocab_size) == {5} def test_token_tag_dispatch_trigger_loop(): """Token trigger with loop_after_dispatch returns to start after body completes.""" vocab = ["", "", "hello", "trigger", "content"] grammar_str = """ body ::= Token(4) root ::= TokenTagDispatch( (3, body), loop_after_dispatch=true )""" matcher, ti, bitmask = _make_bitmask_helper(vocab, grammar_str) assert matcher.accept_token(3) # trigger dispatches to body assert matcher.accept_token(4) # Token(4) completes body assert _get_accepted(matcher, bitmask, ti.vocab_size) == {0, 1, 2, 3, 4} def test_token_tag_dispatch_trigger_and_exclude_no_overlap(): """Token trigger IDs and excludes must not overlap.""" grammar_str = """ body ::= Token(2) root ::= TokenTagDispatch( (3, body), excludes=(3,) )""" with pytest.raises(Exception): xgr.Grammar.from_ebnf(grammar_str) def test_token_tag_dispatch_trigger_in_bitmask(): """Trigger tokens accepted via kToken edge, others via ExcludeToken self-loop.""" vocab = ["", "", "hello", "trigger", "content"] grammar_str = """ body ::= Token(4) root ::= TokenTagDispatch( (3, body) )""" matcher, ti, bitmask = _make_bitmask_helper(vocab, grammar_str) assert _get_accepted(matcher, bitmask, ti.vocab_size) == {0, 1, 2, 3, 4} assert matcher.accept_token(3) # dispatch trigger assert _get_accepted(matcher, bitmask, ti.vocab_size) == {4} def test_token_tag_dispatch_full_combo(): """Token triggers + excludes all working together.""" vocab = ["", "", "hello", "B", "", "content", "blocked"] grammar_str = """ tool_body ::= Token(5) other_body ::= Token(5) root ::= TokenTagDispatch( (3, tool_body), (4, other_body), excludes=(6,) )""" matcher, ti, bitmask = _make_bitmask_helper(vocab, grammar_str) assert _get_accepted(matcher, bitmask, ti.vocab_size) == {0, 1, 2, 3, 4, 5} # --- Lookahead Assertion + kToken tests --- def test_lookahead_exact_with_token_set(): """Exact lookahead containing kToken: tokens matching the rule are accepted.""" vocab = ["", "", "abc", "abcd", "X"] tokenizer_info = xgr.TokenizerInfo(vocab) compiled = xgr.GrammarCompiler(tokenizer_info).compile_grammar( """ rule_a ::= [a-z]+ root ::= rule_a Token(4) """ ) matcher = xgr.GrammarMatcher(compiled) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(token_bitmask) rejected = set(_get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size)) assert rejected == {0, 1, 4} def test_lookahead_token_set_suffix_nonempty_rejected(): """Token that partially matches rule but has bytes left at kToken boundary → rejected.""" vocab = ["", "", "ab", "a", "X"] tokenizer_info = xgr.TokenizerInfo(vocab) compiled = xgr.GrammarCompiler(tokenizer_info).compile_grammar( """ rule_a ::= "a" root ::= rule_a Token(4) """ ) matcher = xgr.GrammarMatcher(compiled) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(token_bitmask) rejected = set(_get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size)) assert rejected == {0, 1, 2, 4} def test_lookahead_mixed_char_and_token(): """Lookahead with char elements before kToken.""" vocab = ["", "", "abc", "abc!", "X"] tokenizer_info = xgr.TokenizerInfo(vocab) compiled = xgr.GrammarCompiler(tokenizer_info).compile_grammar( """ rule_a ::= [a-z]+ root ::= rule_a "!" Token(4) """ ) matcher = xgr.GrammarMatcher(compiled) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(token_bitmask) rejected = set(_get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size)) assert rejected == {0, 1, 4} # --- End-to-end tests --- def test_e2e_complex(): """TokenTagDispatch with two trigger paths: tool (char grammar) and code (Token grammar).""" # fmt: off vocab = [ "", # 0 "", # 1 "", # 2 (trigger -> tool_body) "", # 3 (trigger -> code_body) "", # 4 (excluded) "hello", # 5 "he", # 6 (prefix of "hello") "name", # 7 "val", # 8 "x", # 9 "y", # 10 "{", # 11 "}", # 12 ":", # 13 ",", # 14 "[", # 15 "]", # 16 ";", # 17 "42", # 18 "a:", # 19 (crosses [a-z]+ / ":" boundary) "{a", # 20 (crosses "{" / [a-z]+ boundary) "a}", # 21 (crosses [a-z]+ / "}" boundary) "a;", # 22 (crosses [a-z]+ / ";" boundary) "fn(", # 23 (matches "fn(" exactly) ")", # 24 ] # fmt: on grammar_str = """ value ::= [a-z]+ | [0-9]+ entry ::= [a-z]+ ":" value inner ::= entry (";" entry)* body ::= "{" inner "}" | "[" inner "]" tool_body ::= body ("," body)* arg ::= [a-z]+ call ::= "fn(" Token(9, 10) "," arg ")" code_body ::= call (";" call)* root ::= TokenTagDispatch( (2, tool_body), (3, code_body), excludes=(4,) ) """ ti = xgr.TokenizerInfo(vocab) grammar = xgr.Grammar.from_ebnf(grammar_str) A = set(range(len(vocab))) - {4} AZ = {5, 6, 7, 8, 9, 10} # fmt: off paths = [ [ # tool path: trigger -> nested char grammar with uncertainty tokens (None, A), # initial (2, {11, 15, 20}), # -> body: need "{" or "[" (20, AZ | {13, 19}), # {a -> entry key continues (19, AZ | {18, 21, 22}), # a: -> cross-boundary: key done + ":" (18, {12, 17, 18}), # 42 -> [0-9]+ value; then "}" or ";" (17, AZ | {19}), # ; -> second entry (7, AZ | {13, 19}), # name -> entry key (13, AZ | {18, 21, 22}), # : -> value (8, AZ | {12, 17, 21, 22}), # val -> [a-z]+ value; then "}" or ";" (12, A), # } -> tool_body complete, back to self-loop ], [ # code path: trigger -> text-nested-Token grammar fn(Token,arg) (None, A), # initial (3, {23}), # -> call: need "fn(" (23, {9, 10}), # fn( -> Token(9, 10) position (9, {14}), # x (Token 9) -> need "," (14, AZ), # , -> arg: [a-z]+ (7, AZ | {24}), # name -> arg continues or ")" (24, A), # ) -> call complete, back to self-loop ], ] # fmt: on for steps in paths: matcher = _get_matcher_from_grammar_and_tokenizer_info(grammar, ti) bitmask = xgr.allocate_token_bitmask(1, ti.vocab_size) for token_id, expected in steps: if token_id is not None: assert matcher.accept_token(token_id) assert _get_accepted(matcher, bitmask, ti.vocab_size) == expected assert matcher.accept_token(1) # assert matcher.is_terminated() def test_e2e_nested_dispatch(): """Nested TokenTagDispatch: outer excludes 4, inner excludes 5. Token 4 (o_block): rejected at outer, but accepted inside inner dispatch. Token 5 (i_block): rejected at inner, but accepted at outer dispatch. """ vocab = [ "", # 0 "", # 1 "", # 2 "", # 3 "", # 4 "", # 5 "hello", # 6 "world", # 7 "fn(", # 8 ")", # 9 "x", # 10 "y", # 11 ] grammar_str = """ leaf ::= Token(10, 11) inner ::= TokenTagDispatch((3, leaf), excludes=(5,)) tool_fn ::= "fn(" inner ")" root ::= TokenTagDispatch((2, tool_fn), excludes=(4,)) """ ALL = set(range(len(vocab))) OUTER = ALL - {4} INNER = ALL - {1, 5} ti = xgr.TokenizerInfo(vocab) grammar = xgr.Grammar.from_ebnf(grammar_str) def fresh(): m = _get_matcher_from_grammar_and_tokenizer_info(grammar, ti) b = xgr.allocate_token_bitmask(1, ti.vocab_size) return m, b # fmt: off paths = [ [ # Path A: outer trigger -> fn( -> inner -> leaf -> ) (None, OUTER), (2, {8}), (8, INNER), (6, INNER), (3, {10, 11}), (10, INNER), (9, ALL), (1, None), ], [ # Path B: outer loop only, (4) rejected (None, OUTER), (6, OUTER), (5, OUTER), (4, False), (1, None), ], [ # Path C1: (4) rejected at outer (None, OUTER), (4, False), ], [ # Path C2: (4) accepted inside inner (2, {8}), (8, INNER), (4, INNER), (9, ALL), (1, None), ], ] # fmt: on for steps in paths: m, b = fresh() for token_id, expected in steps: if token_id is None: assert _get_accepted(m, b, ti.vocab_size) == expected elif expected is False: assert not m.accept_token(token_id) elif expected is None: assert m.accept_token(token_id) assert m.is_terminated() else: assert m.accept_token(token_id) assert _get_accepted(m, b, ti.vocab_size) == expected def test_e2e_nested_exclude_loop(): """Nested ExcludeToken-only loop: [a-z]+ loop Token(5) [a-z]+. Token 4 ("###") is excluded from loop AND not [a-z]+ -> always rejected. Token 5 ("") is excluded from loop but accepted via Token(5) after loop. Token 0 ("") is consumed by loop (non-[a-z], non-excluded). """ vocab = ["", "", "hello", "world", "###", "", "foo", "done"] # 0 1 2 3 4 5 6 7 grammar_str = """ loop ::= TokenTagDispatch(excludes=(4, 5)) root ::= [a-z]+ loop Token(5) [a-z]+ """ ti = xgr.TokenizerInfo(vocab) grammar = xgr.Grammar.from_ebnf(grammar_str) AZ = {2, 3, 6, 7} LOOP = {0, 2, 3, 5, 6, 7} # fmt: off paths = [ [ # Main path: [a-z]+ -> loop -> Token(5) -> [a-z]+ -> end (None, AZ), (4, False), (2, LOOP), (0, LOOP), (3, LOOP), (5, AZ), (7, {1} | AZ), (1, None), ], ] # fmt: on for steps in paths: matcher = _get_matcher_from_grammar_and_tokenizer_info(grammar, ti) bitmask = xgr.allocate_token_bitmask(1, ti.vocab_size) for token_id, expected in steps: if token_id is None: assert _get_accepted(matcher, bitmask, ti.vocab_size) == expected elif expected is False: assert not matcher.accept_token(token_id) elif expected is None: assert matcher.accept_token(token_id) assert matcher.is_terminated() else: assert matcher.accept_token(token_id) assert _get_accepted(matcher, bitmask, ti.vocab_size) == expected def test_e2e_mixed_tag_and_token_dispatch(): """Three-layer nesting: TagDispatch -> TokenTagDispatch -> TagDispatch. Outer (TagDispatch): trigger "", excludes string "" Mid (TokenTagDispatch): trigger token 3, excludes token 4 Inner (TagDispatch): trigger "", excludes string "" Key: token 6 ("") rejected by string-based excludes (outer+inner), but temporarily accepted inside mid (token-based, doesn't exclude 6). """ vocab = [ "", # 0 "", # 1 "", # 2 "", # 3 "", # 4 "", # 5 "", # 6 "hello", # 7 "world", # 8 "x", # 9 "y", # 10 "done", # 11 ] grammar_str = """ leaf ::= [a-z]+ inner ::= TagDispatch(("", leaf), excludes=("")) mid_body ::= Token(9, 10) inner mid ::= TokenTagDispatch((3, mid_body), excludes=(4,)) root ::= TagDispatch(("", mid), excludes=("")) """ ALL = set(range(len(vocab))) OUTER = ALL - {6} ti = xgr.TokenizerInfo(vocab) grammar = xgr.Grammar.from_ebnf(grammar_str) def fresh(): m = _get_matcher_from_grammar_and_tokenizer_info(grammar, ti) b = xgr.allocate_token_bitmask(1, ti.vocab_size) return m, b # expected=False means reject; expected=None means accept+terminated # fmt: off paths = [ [ # Path A: full traversal through all 3 layers (None, OUTER), (7, OUTER), (2, ALL), (3, OUTER), (9, OUTER), (8, OUTER), (5, OUTER), (11, OUTER), (1, None), ], [ # Path B: outer loop only, (6) rejected (None, OUTER), (7, OUTER), (8, OUTER), (6, False), (1, None), ], [ # Path C1: (6) rejected at outer (None, OUTER), (6, False), ], [ # Path C2: (6) accepted inside mid (2, ALL), (6, ALL), ], [ # Path C3: (6) rejected by inner excludes (2, ALL), (3, OUTER), (9, OUTER), (6, False), ], [ # Path D1: (4) accepted at outer (4, OUTER), ], [ # Path D2: (4) accepted at inner (2, ALL), (3, OUTER), (9, OUTER), (4, OUTER), ], ] # fmt: on for steps in paths: m, b = fresh() for token_id, expected in steps: if token_id is None: assert _get_accepted(m, b, ti.vocab_size) == expected elif expected is False: assert not m.accept_token(token_id) elif expected is None: assert m.accept_token(token_id) assert m.is_terminated() else: assert m.accept_token(token_id) assert _get_accepted(m, b, ti.vocab_size) == expected def test_rollback(): """Rollback restores mask and accept_token behavior for token edges.""" vocab = ["", "", "", "", "hello", "world", "fn(", ")", "x", "y"] # 0 1 2 3 4 5 6 7 8 9 grammar_str = """ arg ::= [a-z]+ call ::= "fn(" Token(8, 9) "," arg ")" root ::= TokenTagDispatch( (2, call), excludes=(3,) ) """ ti = xgr.TokenizerInfo(vocab) grammar = xgr.Grammar.from_ebnf(grammar_str) m = _get_matcher_from_grammar_and_tokenizer_info(grammar, ti) b = xgr.allocate_token_bitmask(1, ti.vocab_size) mask_0 = _get_accepted(m, b, ti.vocab_size) assert m.accept_token(2) # trigger mask_1 = _get_accepted(m, b, ti.vocab_size) assert m.accept_token(6) # fn( mask_2 = _get_accepted(m, b, ti.vocab_size) assert m.accept_token(8) # x (Token edge) mask_3 = _get_accepted(m, b, ti.vocab_size) # Rollback all 3 tokens m.rollback(3) assert _get_accepted(m, b, ti.vocab_size) == mask_0 # Re-accept and verify masks match assert m.accept_token(2) assert _get_accepted(m, b, ti.vocab_size) == mask_1 assert m.accept_token(6) assert _get_accepted(m, b, ti.vocab_size) == mask_2 assert m.accept_token(8) assert _get_accepted(m, b, ti.vocab_size) == mask_3 # Rollback 2, then continue on a different path m.rollback(2) assert _get_accepted(m, b, ti.vocab_size) == mask_1 assert m.accept_token(6) assert m.accept_token(9) # y instead of x assert _get_accepted(m, b, ti.vocab_size) == mask_3 # same: need "," # Rollback 1 past the token edge, re-accept m.rollback(1) assert _get_accepted(m, b, ti.vocab_size) == mask_2 assert m.accept_token(8) assert _get_accepted(m, b, ti.vocab_size) == mask_3 # --- Structural tag acceptance tests --- STAG_VOCAB = [ "", # 0 "", # 1 "", # 2 "", # 3 "", # 4 "", # 5 "", # 6 "hello", # 7 "world", # 8 "{", # 9 "}", # 10 "fn(", # 11 ")", # 12 "x", # 13 "y", # 14 ",", # 15 "", # 16 ] STAG_STOP = 1 def _stag_matcher(stag_json): ti = xgr.TokenizerInfo(STAG_VOCAB) compiler = xgr.GrammarCompiler(ti) compiled = compiler.compile_structural_tag(stag_json) m = xgr.GrammarMatcher(compiled) b = xgr.allocate_token_bitmask(1, ti.vocab_size) return m, b, ti def _accept_tokens(m, tokens): for t in tokens: ok = m.accept_token(t) assert ok, f"Failed to accept token {t} (vocab: {STAG_VOCAB[t]!r})" def _accept_and_stop(m, tokens): _accept_tokens(m, tokens) assert m.accept_token(STAG_STOP), "Failed to accept stop token" assert m.is_terminated() def test_stag_token_begin_end(): """Tag with token begin/end wrapping JSON content.""" stag = { "type": "structural_tag", "format": { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "const_string", "value": "hello"}, "end": {"type": "token", "token": ""}, }, } m, b, ti = _stag_matcher(stag) _accept_and_stop(m, [2, 7, 4]) # hello def test_stag_exclude_token_basic(): """ExcludeTokenFormat rejects specified tokens and auto-detected end.""" stag = { "type": "structural_tag", "format": { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "exclude_token", "exclude_tokens": [16]}, # exclude "end": {"type": "token", "token": ""}, }, } m, b, ti = _stag_matcher(stag) _accept_tokens(m, [2]) # assert not m.accept_token(16) # excluded assert not m.accept_token(4) # auto-excluded assert m.accept_token(7) # hello accepted _accept_and_stop(m, [4]) # def test_stag_any_tokens_loop(): """AnyTokensFormat accepts arbitrary tokens until end, excluding specified.""" stag = { "type": "structural_tag", "format": { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "any_tokens", "exclude_tokens": [16]}, # exclude "end": {"type": "token", "token": ""}, }, } m, b, ti = _stag_matcher(stag) _accept_tokens(m, [5]) # _accept_tokens(m, [7, 8, 13, 14, 9, 10]) # hello world x y { } assert not m.accept_token(16) # excluded # token 6 () is accepted: ends any_tokens (zero more) then matches end _accept_tokens(m, [0, 3, 2]) # all ok _accept_and_stop(m, [6]) # def test_stag_any_tokens_empty(): """AnyTokensFormat can match zero tokens (go straight to end).""" stag = { "type": "structural_tag", "format": { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "any_tokens"}, "end": {"type": "token", "token": ""}, }, } m, b, ti = _stag_matcher(stag) _accept_and_stop(m, [2, 4]) # (zero content tokens) def test_stag_token_triggered_tags_basic(): """TokenTriggeredTagsFormat dispatches to different tags based on trigger tokens.""" stag = { "type": "structural_tag", "format": { "type": "token_triggered_tags", "trigger_tokens": ["", ""], "tags": [ { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "const_string", "value": "hello"}, "end": {"type": "token", "token": ""}, }, { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "const_string", "value": "world"}, "end": {"type": "token", "token": ""}, }, ], "exclude_tokens": [16], }, } m, b, ti = _stag_matcher(stag) assert not m.accept_token(16) # excluded _accept_tokens(m, [7, 8]) # hello world (free tokens) _accept_tokens(m, [2, 7, 4]) # hello _accept_tokens(m, [13, 14]) # x y (free tokens) _accept_tokens(m, [3, 8, 4]) # world assert m.accept_token(STAG_STOP) assert m.is_terminated() def test_stag_token_triggered_stop_after_first(): """TokenTriggeredTagsFormat with stop_after_first stops after one tag.""" stag = { "type": "structural_tag", "format": { "type": "token_triggered_tags", "trigger_tokens": ["", ""], "tags": [ { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "const_string", "value": "x"}, "end": {"type": "token", "token": ""}, }, { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "const_string", "value": "y"}, "end": {"type": "token", "token": ""}, }, ], "stop_after_first": True, }, } m, b, ti = _stag_matcher(stag) _accept_tokens(m, [7]) # hello (free) _accept_and_stop(m, [2, 13, 4]) # x def test_stag_token_triggered_at_least_one(): """TokenTriggeredTagsFormat with at_least_one+stop_after_first = exactly one tag.""" stag = { "type": "structural_tag", "format": { "type": "token_triggered_tags", "trigger_tokens": [""], "tags": [ { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "const_string", "value": "x"}, "end": {"type": "token", "token": ""}, } ], "at_least_one": True, "stop_after_first": True, }, } m, b, ti = _stag_matcher(stag) # Must start with trigger — free tokens not accepted _accept_and_stop(m, [2, 13, 4]) # x def test_stag_nested_token_tags_with_any_tokens(): """Nested: token_triggered_tags containing tags with any_tokens content. Outer: token_triggered_tags with triggers and . tag: any_tokens content (free tokens until ). tag: sequence of exclude_token + const_string. """ stag = { "type": "structural_tag", "format": { "type": "token_triggered_tags", "trigger_tokens": ["", ""], "tags": [ { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "any_tokens", "exclude_tokens": [16]}, "end": {"type": "token", "token": ""}, }, { "type": "tag", "begin": {"type": "token", "token": ""}, "content": { "type": "sequence", "elements": [ {"type": "exclude_token", "exclude_tokens": [16]}, {"type": "const_string", "value": "x"}, ], }, "end": {"type": "token", "token": ""}, }, ], "exclude_tokens": [16], }, } m, b, ti = _stag_matcher(stag) # Free tokens, then trigger _accept_tokens(m, [7, 8]) # hello world # -> any_tokens (multiple) -> _accept_tokens(m, [2, 9, 10, 13, 14, 7, 4]) # { } x y hello # Free tokens, then trigger _accept_tokens(m, [14]) # y # -> exclude_token (single) + "x" -> _accept_tokens(m, [3, 7, 13, 4]) # hello(=single exclude_token) x assert m.accept_token(STAG_STOP) assert m.is_terminated() def test_stag_sequence_of_token_formats(): """Sequence of multiple token formats and string content.""" stag = { "type": "structural_tag", "format": { "type": "sequence", "elements": [ {"type": "token", "token": ""}, {"type": "const_string", "value": "fn("}, {"type": "exclude_token", "exclude_tokens": [16, 4]}, {"type": "const_string", "value": ")"}, {"type": "token", "token": ""}, ], }, } m, b, ti = _stag_matcher(stag) _accept_tokens(m, [2]) # _accept_tokens(m, [11]) # fn( assert not m.accept_token(16) # excluded assert not m.accept_token(4) # excluded _accept_tokens(m, [7]) # hello (single exclude_token) _accept_tokens(m, [12]) # ) _accept_and_stop(m, [4]) # def test_stag_or_token_and_string_paths(): """Or format choosing between token-level and string-level paths.""" stag = { "type": "structural_tag", "format": { "type": "or", "elements": [ { "type": "sequence", "elements": [ {"type": "token", "token": ""}, {"type": "const_string", "value": "hello"}, ], }, {"type": "const_string", "value": "world"}, ], }, } # Path A: token path m, b, ti = _stag_matcher(stag) _accept_and_stop(m, [2, 7]) # hello # Path B: string path m2, _, _ = _stag_matcher(stag) _accept_and_stop(m2, [8]) # world def test_stag_complex_multi_dispatch(): """Complex: outer triggered_tags (string) wrapping inner token_triggered_tags. Outer: triggered_tags with string trigger "" dispatching to a tag whose content is a token_triggered_tags. Inner: token_triggered_tags with triggers and , each wrapping any_tokens content. """ stag = { "type": "structural_tag", "format": { "type": "triggered_tags", "triggers": [""], "tags": [ { "type": "tag", "begin": "", "content": { "type": "token_triggered_tags", "trigger_tokens": ["", ""], "tags": [ { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "any_tokens"}, "end": {"type": "token", "token": ""}, }, { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "any_tokens", "exclude_tokens": [16]}, "end": {"type": "token", "token": ""}, }, ], }, "end": "", } ], }, } m, b, ti = _stag_matcher(stag) # Free text before outer trigger _accept_tokens(m, [7, 8]) # hello world # Outer trigger: (string match) _accept_tokens(m, [2]) # # Now inside token_triggered_tags: free tokens (any except triggers) _accept_tokens(m, [13, 14]) # x y # Inner trigger: -> any_tokens -> _accept_tokens(m, [3, 7, 8, 9, 10, 4]) # hello world { } # More free tokens inside token_triggered_tags _accept_tokens(m, [7]) # hello # Inner trigger: -> any_tokens (exclude ) -> _accept_tokens(m, [5, 13, 14, 7]) # x y hello assert not m.accept_token(16) # excluded inside think _accept_tokens(m, [6]) # # End outer tag with string "" _accept_tokens(m, [4]) # (string) # Free text after outer tag _accept_tokens(m, [8]) # world assert m.accept_token(STAG_STOP) assert m.is_terminated() def test_stag_star_of_token_sequence(): """Star wrapping a sequence of token + string elements.""" stag = { "type": "structural_tag", "format": { "type": "star", "content": { "type": "sequence", "elements": [ {"type": "token", "token": ""}, {"type": "const_string", "value": "x"}, {"type": "token", "token": ""}, ], }, }, } m, b, ti = _stag_matcher(stag) # Repeat the pattern 3 times for _ in range(3): _accept_tokens(m, [2, 13, 4]) # x assert m.accept_token(STAG_STOP) assert m.is_terminated() def test_stag_star_of_token_sequence_zero(): """Star can match zero repetitions.""" stag = { "type": "structural_tag", "format": { "type": "star", "content": { "type": "sequence", "elements": [ {"type": "token", "token": ""}, {"type": "const_string", "value": "x"}, ], }, }, } m, b, ti = _stag_matcher(stag) assert m.accept_token(STAG_STOP) assert m.is_terminated() def test_stag_multiple_triggered_tags_rounds(): """TokenTriggeredTags: multiple rounds of dispatches interleaved with free tokens.""" stag = { "type": "structural_tag", "format": { "type": "token_triggered_tags", "trigger_tokens": ["", "", ""], "tags": [ { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "const_string", "value": "hello"}, "end": {"type": "token", "token": ""}, }, { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "exclude_token", "exclude_tokens": []}, "end": {"type": "token", "token": ""}, }, { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "any_tokens"}, "end": {"type": "token", "token": ""}, }, ], }, } m, b, ti = _stag_matcher(stag) # Round 1: hello _accept_tokens(m, [2, 7, 4]) # Free tokens _accept_tokens(m, [13, 14, 7]) # x y hello # Round 2: (single token) _accept_tokens(m, [3, 8, 4]) # world # Round 3: any_tokens _accept_tokens(m, [5, 7, 8, 13, 14, 9, 10, 6]) # ... # Round 4: again _accept_tokens(m, [2, 7, 4]) # hello # More free tokens then stop _accept_tokens(m, [8]) # world assert m.accept_token(STAG_STOP) assert m.is_terminated() def test_stag_exclude_token_with_string_excludes(): """ExcludeTokenFormat with string-based token references in exclude_tokens.""" stag = { "type": "structural_tag", "format": { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "exclude_token", "exclude_tokens": ["", "", ""]}, "end": {"type": "token", "token": ""}, }, } m, b, ti = _stag_matcher(stag) _accept_tokens(m, [2]) # assert not m.accept_token(16) # excluded assert not m.accept_token(4) # excluded assert not m.accept_token(5) # excluded _accept_tokens(m, [7]) # hello accepted _accept_and_stop(m, [4]) # def test_stag_token_triggered_string_token_refs(): """TokenTriggeredTags using string references for trigger_tokens and exclude_tokens.""" stag = { "type": "structural_tag", "format": { "type": "token_triggered_tags", "trigger_tokens": ["", ""], "tags": [ { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "any_tokens", "exclude_tokens": [""]}, "end": {"type": "token", "token": ""}, }, { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "const_string", "value": "y"}, "end": {"type": "token", "token": ""}, }, ], "exclude_tokens": [""], }, } m, b, ti = _stag_matcher(stag) assert not m.accept_token(16) # excluded at top level _accept_tokens(m, [7]) # hello # -> any_tokens (exclude ) -> _accept_tokens(m, [2]) # assert not m.accept_token(16) # excluded inside any_tokens _accept_tokens(m, [8, 13, 4]) # world x # -> "y" -> _accept_tokens(m, [3, 14, 4]) # y assert m.accept_token(STAG_STOP) assert m.is_terminated() def test_stag_tag_with_sequence_content_mixed(): """Tag content is a complex sequence: token + exclude_token + const_string + any_tokens.""" stag = { "type": "structural_tag", "format": { "type": "tag", "begin": {"type": "token", "token": ""}, "content": { "type": "sequence", "elements": [ {"type": "token", "token": ""}, {"type": "exclude_token", "exclude_tokens": [16]}, {"type": "const_string", "value": "x"}, {"type": "any_tokens", "exclude_tokens": [16]}, ], }, "end": {"type": "token", "token": ""}, }, } m, b, ti = _stag_matcher(stag) _accept_tokens(m, [2]) # _accept_tokens(m, [3]) # (token) assert not m.accept_token(16) # excluded _accept_tokens(m, [7]) # hello (single exclude_token) _accept_tokens(m, [13]) # x (const_string) assert not m.accept_token(16) # excluded in any_tokens _accept_tokens(m, [8, 14, 5]) # world y (any_tokens loop) _accept_and_stop(m, [4]) # def test_stag_or_between_token_triggered_and_string_triggered(): """Or between token_triggered_tags and triggered_tags paths.""" stag = { "type": "structural_tag", "format": { "type": "or", "elements": [ { "type": "token_triggered_tags", "trigger_tokens": [""], "tags": [ { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "const_string", "value": "hello"}, "end": {"type": "token", "token": ""}, } ], "stop_after_first": True, "at_least_one": True, }, { "type": "triggered_tags", "triggers": ["fn("], "tags": [ {"type": "tag", "begin": "fn(", "content": {"type": "any_text"}, "end": ")"} ], "stop_after_first": True, "at_least_one": True, }, ], }, } # Path A: token triggered m, b, ti = _stag_matcher(stag) _accept_and_stop(m, [2, 7, 4]) # hello # Path B: string triggered m2, _, _ = _stag_matcher(stag) _accept_tokens(m2, [11]) # fn( _accept_tokens(m2, [7, 8]) # hello world (any_text) _accept_and_stop(m2, [12]) # ) def test_stag_deeply_nested_three_layers(): """Three layers: triggered_tags -> tag -> token_triggered_tags -> tag -> any_tokens. Layer 1: String triggered_tags with trigger "fn(" Layer 2: Tag with string begin "fn(" wrapping token_triggered_tags Layer 3: Token triggered tag wrapping any_tokens, and wrapping exclude_token """ stag = { "type": "structural_tag", "format": { "type": "triggered_tags", "triggers": ["fn("], "tags": [ { "type": "tag", "begin": "fn(", "content": { "type": "token_triggered_tags", "trigger_tokens": ["", ""], "tags": [ { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "any_tokens", "exclude_tokens": [""]}, "end": {"type": "token", "token": ""}, }, { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "exclude_token", "exclude_tokens": [""]}, "end": {"type": "token", "token": ""}, }, ], "exclude_tokens": [""], }, "end": ")", } ], }, } m, b, ti = _stag_matcher(stag) # Free text before string trigger _accept_tokens(m, [7, 8]) # hello world # String trigger: fn( _accept_tokens(m, [11]) # fn( # Inside token_triggered_tags: free tokens _accept_tokens(m, [13]) # x assert not m.accept_token(16) # excluded at token_triggered level # Trigger -> any_tokens (exclude ) -> _accept_tokens(m, [5, 7, 8, 13, 14]) # hello world x y assert not m.accept_token(16) # excluded inside any_tokens _accept_tokens(m, [6]) # # More free tokens _accept_tokens(m, [14]) # y # Trigger -> exclude_token (single, exclude ) -> _accept_tokens(m, [3]) # assert not m.accept_token(16) # excluded inside exclude_token _accept_tokens(m, [7]) # hello (single token) _accept_tokens(m, [4]) # # End outer tag with string ")" _accept_tokens(m, [12]) # ) # Free text after _accept_tokens(m, [8]) # world assert m.accept_token(STAG_STOP) assert m.is_terminated() def test_stag_any_tokens_all_excluded_except_end(): """AnyTokensFormat where almost all tokens are excluded, forcing direct end.""" stag = { "type": "structural_tag", "format": { "type": "tag", "begin": {"type": "token", "token": ""}, "content": { "type": "any_tokens", "exclude_tokens": [0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], }, "end": {"type": "token", "token": ""}, }, } m, b, ti = _stag_matcher(stag) _accept_tokens(m, [2]) # # Almost everything excluded; only end token (4) should work to proceed _accept_and_stop(m, [4]) # def test_stag_token_tag_with_or_content(): """Tag with token begin/end wrapping or content of different types.""" stag = { "type": "structural_tag", "format": { "type": "tag", "begin": {"type": "token", "token": ""}, "content": { "type": "or", "elements": [ {"type": "const_string", "value": "hello"}, { "type": "sequence", "elements": [ {"type": "exclude_token", "exclude_tokens": [16]}, {"type": "const_string", "value": "world"}, ], }, ], }, "end": {"type": "token", "token": ""}, }, } # Path A: const_string "hello" m, b, ti = _stag_matcher(stag) _accept_and_stop(m, [2, 7, 4]) # hello # Path B: exclude_token + "world" m2, _, _ = _stag_matcher(stag) _accept_tokens(m2, [2]) # _accept_tokens(m2, [13]) # x (single exclude_token) _accept_tokens(m2, [8]) # world _accept_and_stop(m2, [4]) # def test_stag_mixed_begin_end_types(): """Tags with different begin/end type combinations within token_triggered_tags.""" stag = { "type": "structural_tag", "format": { "type": "token_triggered_tags", "trigger_tokens": ["", "", ""], "tags": [ { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "const_string", "value": "hello"}, "end": "", }, { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "exclude_token", "exclude_tokens": [""]}, "end": {"type": "token", "token": ""}, }, { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "any_tokens", "exclude_tokens": [""]}, "end": "", }, ], }, } m, b, ti = _stag_matcher(stag) # Tag 1: token begin, string end _accept_tokens(m, [2]) # _accept_tokens(m, [7]) # hello _accept_tokens(m, [4]) # (as string) # Free tokens _accept_tokens(m, [13]) # x # Tag 2: token begin, token end _accept_tokens(m, [3]) # assert not m.accept_token(16) # excluded _accept_tokens(m, [7]) # hello (single exclude_token) _accept_tokens(m, [4]) # # Tag 3: token begin, string end _accept_tokens(m, [5]) # assert not m.accept_token(16) # excluded _accept_tokens(m, [7, 8, 13]) # hello world x _accept_tokens(m, [6]) # (as string) assert m.accept_token(STAG_STOP) assert m.is_terminated() def test_stag_repeated_token_triggered_tags_different_tags(): """Cycle through all 3 tag types multiple times in token_triggered_tags.""" stag = { "type": "structural_tag", "format": { "type": "token_triggered_tags", "trigger_tokens": ["", "", ""], "tags": [ { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "const_string", "value": "x"}, "end": {"type": "token", "token": ""}, }, { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "const_string", "value": "y"}, "end": {"type": "token", "token": ""}, }, { "type": "tag", "begin": {"type": "token", "token": ""}, "content": {"type": "const_string", "value": "hello"}, "end": {"type": "token", "token": ""}, }, ], }, } m, b, ti = _stag_matcher(stag) for _ in range(3): # x _accept_tokens(m, [2, 13, 4]) # free _accept_tokens(m, [7]) # hello # y _accept_tokens(m, [3, 14, 4]) # hello _accept_tokens(m, [5, 7, 6]) assert m.accept_token(STAG_STOP) assert m.is_terminated() def test_stag_any_tokens_excludes_allow_empty_end(): """Tags with any_tokens content + exclude_tokens should allow end: "". Models a dispatch loop where channels have no explicit end delimiter — the content stops when an excluded trigger token appears, and the dispatch loop handles the transition. Pattern: hello world x y ^trigger ^content ^next trigger Each tag is: begin=, content=any_tokens(exclude=[,]), end="" The content stops naturally when the next appears. The dispatch loop re-triggers. """ stag = { "type": "structural_tag", "format": { "type": "token_triggered_tags", "trigger_tokens": [""], "tags": [ { "begin": {"type": "token", "token": ""}, "content": {"type": "any_tokens", "exclude_tokens": ["", ""]}, "end": "", } ], }, } m, b, ti = _stag_matcher(stag) # First dispatch: hello world _accept_tokens(m, [2, 7, 8]) # hello world # Second dispatch: x y _accept_tokens(m, [2, 13, 14]) # x y # Stop assert m.accept_token(STAG_STOP) assert m.is_terminated() def test_stag_any_tokens_exclude_redispatch(): """any_tokens(exclude=[]) stops content when appears, allowing re-dispatch.""" stag = { "type": "structural_tag", "format": { "type": "token_triggered_tags", "trigger_tokens": [""], "tags": [ { "begin": {"type": "token", "token": ""}, "content": {"type": "any_tokens", "exclude_tokens": [""]}, "end": "", } ], }, } m, b, ti = _stag_matcher(stag) # First tag: hello _accept_tokens(m, [2, 7]) # hello # re-triggers (content excluded , so dispatch takes over) _accept_tokens(m, [2, 8]) # world # Third round then stop _accept_tokens(m, [2, 13]) # x assert m.accept_token(STAG_STOP) assert m.is_terminated() if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.2.3/tests/python/test_tokenizer_info.py000066400000000000000000000257721521764210300224710ustar00rootroot00000000000000import logging import sys from typing import Dict, List, Tuple import pytest from transformers import AutoTokenizer, PreTrainedTokenizerBase import xgrammar as xgr @pytest.fixture(scope="module") def tokenizer_info_storage() -> Dict[str, Tuple[PreTrainedTokenizerBase, xgr.TokenizerInfo]]: """Mapping from the tokenizer path to the huggingface tokenizer and XGrammar tokenizer info.""" return {} tokenizer_path__vocab_type__prepend_space = [ ("luodian/llama-7b-hf", xgr.VocabType.BYTE_FALLBACK, True), ("meta-llama/Llama-2-7b-chat-hf", xgr.VocabType.BYTE_FALLBACK, True), ("meta-llama/Meta-Llama-3-8B-Instruct", xgr.VocabType.BYTE_LEVEL, False), ("meta-llama/Meta-Llama-3.1-8B-Instruct", xgr.VocabType.BYTE_LEVEL, False), ("lmsys/vicuna-7b-v1.5", xgr.VocabType.BYTE_FALLBACK, True), ("NousResearch/Hermes-2-Theta-Llama-3-70B", xgr.VocabType.BYTE_LEVEL, False), ("NousResearch/Hermes-3-Llama-3.1-8B", xgr.VocabType.BYTE_LEVEL, False), ("google/gemma-2b-it", xgr.VocabType.BYTE_FALLBACK, False), ("CohereForAI/aya-23-8B", xgr.VocabType.BYTE_LEVEL, False), ("deepseek-ai/DeepSeek-Coder-V2-Instruct", xgr.VocabType.BYTE_LEVEL, False), ("deepseek-ai/DeepSeek-V2-Chat-0628", xgr.VocabType.BYTE_LEVEL, False), ("deepseek-ai/deepseek-coder-7b-instruct-v1.5", xgr.VocabType.BYTE_LEVEL, False), ("microsoft/phi-2", xgr.VocabType.BYTE_LEVEL, False), ("microsoft/Phi-3-mini-4k-instruct", xgr.VocabType.BYTE_FALLBACK, True), ("microsoft/Phi-3.5-mini-instruct", xgr.VocabType.BYTE_FALLBACK, True), ("Qwen/Qwen1.5-4B-Chat", xgr.VocabType.BYTE_LEVEL, False), ("Qwen/Qwen2-7B-Instruct", xgr.VocabType.BYTE_LEVEL, False), ("microsoft/Phi-3-small-8k-instruct", xgr.VocabType.RAW, False), ("Qwen/Qwen-7B-Chat", xgr.VocabType.RAW, False), ("meta-llama/Llama-3.2-1B", xgr.VocabType.BYTE_LEVEL, False), ("google/gemma-2-2b-it", xgr.VocabType.BYTE_FALLBACK, False), ("deepseek-ai/DeepSeek-V2.5", xgr.VocabType.BYTE_LEVEL, False), ("Qwen/Qwen2.5-1.5B", xgr.VocabType.BYTE_LEVEL, False), ("internlm/internlm2_5-7b-chat", xgr.VocabType.BYTE_FALLBACK, False), ("mistralai/Mixtral-8x22B-Instruct-v0.1", xgr.VocabType.BYTE_FALLBACK, True), ("THUDM/glm-4-9b-chat", xgr.VocabType.RAW, False), ("THUDM/chatglm3-6b", xgr.VocabType.BYTE_FALLBACK, True), ("deepseek-ai/DeepSeek-R1", xgr.VocabType.BYTE_LEVEL, False), ("deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", xgr.VocabType.BYTE_LEVEL, False), ("deepseek-ai/DeepSeek-R1-Distill-Llama-8B", xgr.VocabType.BYTE_LEVEL, False), ("openGPT-X/Teuken-7B-instruct-v0.6", xgr.VocabType.BYTE_FALLBACK, True), ("moonshotai/Kimi-K2-Instruct", xgr.VocabType.BYTE_LEVEL, False), ] tokenizer_paths = [path for path, *_ in tokenizer_path__vocab_type__prepend_space] @pytest.mark.hf_token_required @pytest.mark.parametrize("tokenizer_path", tokenizer_paths) def test_build_tokenizer_info( tokenizer_path: str, tokenizer_info_storage: Dict[str, Tuple[PreTrainedTokenizerBase, xgr.TokenizerInfo]], ): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) tokenizer_info_storage[tokenizer_path] = (tokenizer, tokenizer_info) @pytest.mark.hf_token_required @pytest.mark.parametrize( "tokenizer_path, vocab_type, add_prefix_space", tokenizer_path__vocab_type__prepend_space ) def test_properties( tokenizer_path: str, vocab_type: xgr.VocabType, add_prefix_space: bool, tokenizer_info_storage: Dict[str, Tuple[PreTrainedTokenizerBase, xgr.TokenizerInfo]], ): tokenizer, tokenizer_info = tokenizer_info_storage[tokenizer_path] vocab_dict = tokenizer.get_vocab() max_id = max(vocab_dict.values()) if vocab_dict else -1 assert tokenizer_info.vocab_size == max(len(vocab_dict), max_id + 1) assert tokenizer_info.vocab_type == vocab_type assert tokenizer_info.add_prefix_space == add_prefix_space @pytest.mark.hf_token_required @pytest.mark.parametrize("tokenizer_path", tokenizer_paths) def test_decoded_vocab( tokenizer_path: str, tokenizer_info_storage: Dict[str, Tuple[PreTrainedTokenizerBase, xgr.TokenizerInfo]], ): tokenizer, tokenizer_info = tokenizer_info_storage[tokenizer_path] decoded_vocab = tokenizer_info.decoded_vocab vocab_dict = tokenizer.get_vocab() max_id = max(vocab_dict.values()) if vocab_dict else -1 assert isinstance(decoded_vocab, list) assert all(isinstance(token, bytes) for token in decoded_vocab) assert len(decoded_vocab) == max(len(vocab_dict), max_id + 1) assert len(decoded_vocab) == tokenizer_info.vocab_size @pytest.mark.hf_token_required @pytest.mark.parametrize("tokenizer_path", tokenizer_paths) def test_stop_token_ids( tokenizer_path: str, tokenizer_info_storage: Dict[str, Tuple[PreTrainedTokenizerBase, xgr.TokenizerInfo]], ): tokenizer, tokenizer_info = tokenizer_info_storage[tokenizer_path] if hasattr(tokenizer, "eos_token_id") and tokenizer.eos_token_id is not None: assert tokenizer_info.stop_token_ids == [tokenizer.eos_token_id] else: logging.warning(f"EOS token id is not defined for tokenizer {tokenizer_path}") @pytest.mark.hf_token_required @pytest.mark.parametrize("tokenizer_path", tokenizer_paths) def test_decode_text( tokenizer_path: str, tokenizer_info_storage: Dict[str, Tuple[PreTrainedTokenizerBase, xgr.TokenizerInfo]], ): text = ( "Hello 你好 こんにちは 안녕하세요! 🌎🌍🌏 \u0300\u0301\u0302 \U0001f600\U0001f601\U0001f602 " + "αβγδ АБВГД عربي עברית" + "\n\t\r Special chars: &*()_+-=[]{}|;:'\",.<>?/\\~`!@#$%^haha" ) tokenizer, tokenizer_info = tokenizer_info_storage[tokenizer_path] decoded_vocab = tokenizer_info.decoded_vocab tokenized_text = tokenizer.encode(text) recovered_text = b"".join(decoded_vocab[token_id] for token_id in tokenized_text).decode( "utf-8" ) trial_text = "a" trial_text_roundtrip = b"".join( decoded_vocab[token_id] for token_id in tokenizer.encode(trial_text) ).decode("utf-8") assert trial_text_roundtrip[-1] == "a" detected_prefix = trial_text_roundtrip[:-1] assert tokenizer_info.add_prefix_space == ( len(detected_prefix) > 0 and detected_prefix[-1] == " " ) assert detected_prefix + text == recovered_text tokenizer_path__token_ids__raw_tokens = [ # raw ("microsoft/Phi-3-small-8k-instruct", [10, 94, 37046], [b"+", b"\xa1", b"\xe6\x88\x91"]), # byte_fallback ( "meta-llama/Llama-2-7b-chat-hf", [4, 259, 261, 20565], [b"\x01", b" ", b"er", " исследова".encode("utf-8")], ), # byte_level ( "meta-llama/Meta-Llama-3-8B-Instruct", [1, 37046, 40508], [b'"', "我".encode("utf-8"), b" automotive"], ), ] @pytest.mark.hf_token_required @pytest.mark.parametrize( "tokenizer_path, token_ids, raw_tokens", tokenizer_path__token_ids__raw_tokens ) def test_vocab_conversion(tokenizer_path: str, token_ids: List[int], raw_tokens: List[bytes]): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) vocab = tokenizer_info.decoded_vocab for token_id, raw_token in zip(token_ids, raw_tokens): assert vocab[token_id] == raw_token tokenizer_path__metadata_str = [ ( "microsoft/Phi-3-small-8k-instruct", '{"vocab_type":0,"vocab_size":100352,"add_prefix_space":false,"stop_token_ids":[100257]}', ), ( "meta-llama/Llama-2-7b-chat-hf", '{"vocab_type":1,"vocab_size":32000,"add_prefix_space":true,"stop_token_ids":[2]}', ), ( "meta-llama/Meta-Llama-3-8B-Instruct", '{"vocab_type":2,"vocab_size":128256,"add_prefix_space":false,"stop_token_ids":[128009]}', ), ] @pytest.mark.hf_token_required @pytest.mark.parametrize("tokenizer_path, metadata_str", tokenizer_path__metadata_str) def test_dump_metadata_load(tokenizer_path: str, metadata_str: str): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) assert tokenizer_info.dump_metadata() == metadata_str encoded_vocab = tokenizer.get_vocab() encoded_vocab = [token for token, _ in sorted(encoded_vocab.items(), key=lambda x: x[1])] loaded = xgr.TokenizerInfo.from_vocab_and_metadata(encoded_vocab, metadata_str) assert loaded.decoded_vocab == tokenizer_info.decoded_vocab loaded_new = xgr.TokenizerInfo(tokenizer_info.decoded_vocab) assert loaded_new.decoded_vocab == tokenizer_info.decoded_vocab def test_special_token_detection(): # Now only empty string "" is treated as a special token. vocab_dict = ["", "", "", "[@BOS@]", "regular", "<>", "", ""] tokenizer_info = xgr.TokenizerInfo.from_vocab_and_metadata( vocab_dict, '{"vocab_type":1,"vocab_size":8,"add_prefix_space":true,"stop_token_ids":[2]}' ) expected_special_tokens = {0} assert set(tokenizer_info.special_token_ids) == expected_special_tokens @pytest.mark.hf_token_required @pytest.mark.parametrize( "tokenizer_path", ["meta-llama/Llama-2-7b-chat-hf", "meta-llama/Meta-Llama-3-8B-Instruct"] ) def test_customize_stop_token_ids(tokenizer_path: str): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, stop_token_ids=[1, 2, 3]) assert tokenizer_info.stop_token_ids == [1, 2, 3] @pytest.mark.hf_token_required @pytest.mark.parametrize( "tokenizer_path", ["meta-llama/Llama-2-7b-chat-hf", "meta-llama/Meta-Llama-3-8B-Instruct"] ) def test_padding_vocab_size(tokenizer_path: str): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) original_vocab_size = len(tokenizer.get_vocab()) tokenizer_info = xgr.TokenizerInfo.from_huggingface( tokenizer, vocab_size=original_vocab_size + 5 ) assert tokenizer_info.vocab_size == original_vocab_size + 5 assert tokenizer_info.special_token_ids[-5:] == [original_vocab_size + i for i in range(5)] tokenizer_path__model_vocab_size = [ ("meta-llama/Llama-3.2-11B-Vision-Instruct", 128256), ("meta-llama/Llama-Guard-3-11B-Vision", 128256), ("allenai/Molmo-72B-0924", 152064), ] @pytest.mark.hf_token_required @pytest.mark.parametrize("tokenizer_path, model_vocab_size", tokenizer_path__model_vocab_size) def test_model_vocab_size_smaller_than_tokenizer(tokenizer_path: str, model_vocab_size: int): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) original_vocab_size = len(tokenizer.get_vocab()) assert original_vocab_size > model_vocab_size tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=model_vocab_size) assert tokenizer_info.vocab_size == model_vocab_size assert len(tokenizer_info.decoded_vocab) == model_vocab_size print(tokenizer_info.special_token_ids) print(len(tokenizer_info.decoded_vocab)) if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.2.3/web/000077500000000000000000000000001521764210300141105ustar00rootroot00000000000000xgrammar-0.2.3/web/.gitignore000066400000000000000000000001531521764210300160770ustar00rootroot00000000000000src/xgrammar_binding.js build node_modules dist lib .cache .vscode .parcel-cache example/package-lock.json xgrammar-0.2.3/web/README.md000066400000000000000000000020651521764210300153720ustar00rootroot00000000000000# web-xgrammar This folder contains the source code and emcc bindings for compiling XGrammar to Javascript/Typescript via [emscripten](https://emscripten.org/). ### Build from source 1. Install [emscripten](https://emscripten.org). It is an LLVM-based compiler that compiles C/C++ source code to WebAssembly. - Follow the [installation instruction](https://emscripten.org/docs/getting_started/downloads.html#installation-instructions-using-the-emsdk-recommended) to install the latest emsdk. - Source `emsdk_env.sh` by `source /path/to/emsdk_env.sh`, so that `emcc` is reachable from PATH and the command `emcc` works. - We can verify the successful installation by trying out `emcc` in the terminal. 2. Modify the content of `cmake/config.cmake` to be `web/config.cmake`. 3. Run the following ```bash source /path/to/emsdk_env.sh npm install npm run build ``` ### Example To try out the test webpage, run the following ```bash cd example npm install npm start ``` ### Testing For testing in `node` environment, run: ```bash npm test ``` xgrammar-0.2.3/web/build.sh000077500000000000000000000010741521764210300155500ustar00rootroot00000000000000#!/bin/bash set -euxo pipefail mkdir -p build cd build emcmake cmake ../.. -DXGRAMMAR_BUILD_PYTHON_BINDINGS=OFF\ -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-O3 -DCOMPILE_WASM_RUNTIME -DXGRAMMAR_LOG_CUSTOMIZE=1" emmake make xgrammar -j8 cd .. emcc -o src/xgrammar_binding.js src/xgrammar_binding.cc\ build/libxgrammar.a -lembind\ -O3 -s EXPORT_ES6=1 -s ERROR_ON_UNDEFINED_SYMBOLS=0 -s NO_DYNAMIC_EXECUTION=1 -s MODULARIZE=1 -s SINGLE_FILE=1 -s EXPORTED_RUNTIME_METHODS=FS -s ALLOW_MEMORY_GROWTH=1\ -I../include -I../3rdparty/picojson -I../3rdparty/dlpack/include xgrammar-0.2.3/web/config.cmake000066400000000000000000000001571521764210300163620ustar00rootroot00000000000000set(CMAKE_BUILD_TYPE RelWithDebInfo) set(XGRAMMAR_BUILD_PYTHON_BINDINGS OFF) set(XGRAMMAR_BUILD_CXX_TESTS OFF) xgrammar-0.2.3/web/eslint.config.cjs000066400000000000000000000016401521764210300173540ustar00rootroot00000000000000const { defineConfig, globalIgnores, } = require("eslint/config"); const tsParser = require("@typescript-eslint/parser"); const typescriptEslint = require("@typescript-eslint/eslint-plugin"); const js = require("@eslint/js"); const { FlatCompat, } = require("@eslint/eslintrc"); const compat = new FlatCompat({ baseDirectory: __dirname, recommendedConfig: js.configs.recommended, allConfig: js.configs.all }); module.exports = defineConfig([{ extends: compat.extends("eslint:recommended", "plugin:@typescript-eslint/recommended"), languageOptions: { parser: tsParser, }, plugins: { "@typescript-eslint": typescriptEslint, }, rules: { "@typescript-eslint/no-explicit-any": "off", }, }, globalIgnores([ "**/dist", "**/debug", "**/lib", "**/build", "**/node_modules", "**/xgrammar_binding.js", "**/.eslintrc.cjs", ])]); xgrammar-0.2.3/web/example/000077500000000000000000000000001521764210300155435ustar00rootroot00000000000000xgrammar-0.2.3/web/example/package.json000066400000000000000000000010621521764210300200300ustar00rootroot00000000000000{ "name": "web-xgrammar-example", "version": "0.1.27", "private": true, "type": "module", "scripts": { "start": "parcel src/example.html --port 8888" }, "browser": {}, "devDependencies": { "@mlc-ai/web-xgrammar": "^0.1.27", "@mlc-ai/web-tokenizers": "^0.1.6", "buffer": "^5.7.1", "parcel": "^2.8.3", "process": "^0.11.10", "punycode": "^1.4.1", "querystring-es3": "^0.2.1", "tslib": "^2.3.1", "typescript": "^4.9.5", "url": "^0.11.0" } } xgrammar-0.2.3/web/example/src/000077500000000000000000000000001521764210300163325ustar00rootroot00000000000000xgrammar-0.2.3/web/example/src/example.html000066400000000000000000000002501521764210300206500ustar00rootroot00000000000000

Tokenizer Test Page

Open console to see output xgrammar-0.2.3/web/example/src/example.ts000066400000000000000000000215321521764210300203400ustar00rootroot00000000000000import { GrammarMatcher, TokenizerInfo, GrammarCompiler, CompiledGrammar, Testings, StructuralTag } from "@mlc-ai/web-xgrammar" import { Tokenizer } from "@mlc-ai/web-tokenizers"; import { Type, Static } from "@sinclair/typebox"; async function getTokenizerInfoAndTokenizerFromUrl( tokenizerUrl: string, vocabType: string, prependSpaceInTokenization: boolean, ): Promise<[TokenizerInfo, Tokenizer]> { // 1. Get tokenizer, we use "@mlc-ai/web-tokenizers" here, but any should work const jsonBuffer = await (await fetch(tokenizerUrl)).arrayBuffer(); const tokenizer = await Tokenizer.fromJSON(jsonBuffer); // 2. Get encoded vocab const tstartGetToken = performance.now(); const rawTokenTable: string[] = []; const vocabSize = tokenizer.getVocabSize(); for (let tokenId = 0; tokenId < vocabSize; tokenId++) { rawTokenTable.push(tokenizer.idToToken(tokenId)); } console.log("Get raw token table (ms): ", (performance.now() - tstartGetToken)); // 3. Post process vocab const tstartGetTokenizerInfo = performance.now(); const tokenizerInfo = await TokenizerInfo.createTokenizerInfo(rawTokenTable, vocabType, prependSpaceInTokenization); console.log("createTokenizerInfo (ms): ", (performance.now() - tstartGetTokenizerInfo)); return [tokenizerInfo, tokenizer]; } async function jsonExample() { console.log("Running JSON Example"); const result = await getTokenizerInfoAndTokenizerFromUrl( "https://huggingface.co/mlc-ai/Llama-3.2-1B-Instruct-q4f16_0-MLC/raw/main/tokenizer.json", "byte_level", false, ); const tokenizerInfo = result[0]; const tokenizer = result[1]; const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); // 1. Initialize grammar state matcher with JSON grammar const grammar: CompiledGrammar = await compiler.compileBuiltinJSONGrammar(); const grammarMatcher = await GrammarMatcher.createGrammarMatcher(grammar); console.log(grammarMatcher); // 2. Simulated generation of an LLM const input = String.raw`{"hi": 1}<|end_of_text|>`; const encodedTokens = tokenizer.encode(input); // 3. We expect the matcher to accept all tokens generated since it is a valid JSON for (let i = 0; i < encodedTokens.length; i++) { // 3.1 Generate token bitmask that will modify logits of the LLM if (!grammarMatcher.isTerminated()) { const bitmask = await grammarMatcher.getNextTokenBitmask(); // For debugging, we can check the rejected token IDs from the mask // eslint-disable-next-line @typescript-eslint/no-unused-vars const rejectedIDs = await Testings.debugGetMaskedTokensFromBitmask( bitmask, tokenizerInfo.getVocabSize() ); } // 3.2 Say the LLM generated `curToken`, which is simulated here, we use `acceptToken()` // to update the state of the matcher, so it will generate a new bitmask for the next // auto-regressive generation const curToken = encodedTokens[i]; const accepted = grammarMatcher.acceptToken(curToken); if (!accepted) { throw Error("Expect token to be accepted"); } } // 4. The last token is and stop token, so the matcher has terminated. console.log("grammarMatcher.isTerminated(): ", grammarMatcher.isTerminated()); grammarMatcher.dispose(); } async function jsonSchemaExample() { console.log("Running JSON Schema Example"); // 0. Prepare a schema const T = Type.Object({ name: Type.String(), house: Type.Enum({ Gryffindor: "Gryffindor", Hufflepuff: "Hufflepuff", Ravenclaw: "Ravenclaw", Slytherin: "Slytherin", }), blood_status: Type.Enum({ "Pure-blood": "Pure-blood", "Half-blood": "Half-blood", "Muggle-born": "Muggle-born", }), occupation: Type.Enum({ Student: "Student", Professor: "Professor", "Ministry of Magic": "Ministry of Magic", Other: "Other", }), wand: Type.Object({ wood: Type.String(), core: Type.String(), length: Type.Number(), }), alive: Type.Boolean(), patronus: Type.String(), }); type T = Static; const schema = JSON.stringify(T); console.log("schema: ", schema); const result = await getTokenizerInfoAndTokenizerFromUrl( "https://huggingface.co/mlc-ai/Llama-3.2-1B-Instruct-q4f16_0-MLC/raw/main/tokenizer.json", "byte_level", false, ); const tokenizerInfo = result[0]; const tokenizer = result[1]; // 1. Instantiate matcher with a grammar defined by the above schema const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); const tstartInitMatcher = performance.now(); const grammar: CompiledGrammar = await compiler.compileJSONSchema(schema); const grammarMatcher = await GrammarMatcher.createGrammarMatcher(grammar); console.log("createGrammarMatcher (ms): ", (performance.now() - tstartInitMatcher)); console.log(grammarMatcher); // 2. Simulated generation of an LLM const input = String.raw`{ "name": "Hermione Granger", "house": "Ravenclaw", "blood_status": "Muggle-born", "occupation": "Student", "wand": { "wood": "Vine", "core": "Phoenix Feather", "length": 10 }, "alive": true, "patronus": "Otter" }<|end_of_text|>`; const encodedTokens = tokenizer.encode(input); // 3. We expect the matcher to accept all tokens generated since it is a valid JSON for (let i = 0; i < encodedTokens.length; i++) { // 3.1 Generate token bitmask that will modify logits of the LLM if (!grammarMatcher.isTerminated()) { const bitmask = await grammarMatcher.getNextTokenBitmask(); // For debugging, we can check the rejected token IDs from the mask // eslint-disable-next-line @typescript-eslint/no-unused-vars const rejectedIDs = await Testings.debugGetMaskedTokensFromBitmask( bitmask, tokenizerInfo.getVocabSize() ); } // 3.2 Say the LLM generated `curToken`, which is simulated here, we use `acceptToken()` // to update the state of the matcher, so it will generate a new bitmask for the next // auto-regressive generation const curToken = encodedTokens[i]; const accepted = grammarMatcher.acceptToken(curToken); if (!accepted) { throw Error("Expect token to be accepted"); } } // 4. The last token is and stop token, so the matcher has terminated. console.log("grammarMatcher.isTerminated(): ", grammarMatcher.isTerminated()); grammarMatcher.dispose(); } async function structuralTagExample() { console.log("Running Structural Tag Example"); // 1. Define the schema for our function const weatherSchema = Type.Object({ city: Type.String(), is_celsius: Type.Boolean() }); // 2. Create structural tag for our function const structuralTag: StructuralTag = { type: "structural_tag", format: { type: "triggered_tags", triggers: ["", end: "", content: { type: "json_schema", json_schema: weatherSchema }, }, ], }, }; // 3. Load tokenizer const result = await getTokenizerInfoAndTokenizerFromUrl( "https://huggingface.co/mlc-ai/Llama-3.2-1B-Instruct-q4f16_0-MLC/raw/main/tokenizer.json", "byte_level", false, ); const tokenizerInfo = result[0]; const tokenizer = result[1]; // 4. Get encoded vocabulary const encodedVocab = []; const vocabSize = tokenizer.getVocabSize(); for (let tokenId = 0; tokenId < vocabSize; tokenId++) { encodedVocab.push(tokenizer.idToToken(tokenId)); } // 5. Create compiler and compile the structural tag grammar const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); const compiledGrammar = await compiler.compileStructuralTag(structuralTag); // 6. Create the grammar matcher const matcher = await GrammarMatcher.createGrammarMatcher(compiledGrammar); // 7. Test with sample input const testInput = `I need to check the weather.{"city": "New York", "is_celsius": false} Thanks!`; console.log("Testing with input:", testInput); // 8. Process the input character by character const encodedTokens = tokenizer.encode(testInput); for (let i = 0; i < encodedTokens.length; i++) { const bitmask = await matcher.getNextTokenBitmask(); // For debugging, we can check the rejected token IDs from the mask // eslint-disable-next-line @typescript-eslint/no-unused-vars const rejectedIDs = await Testings.debugGetMaskedTokensFromBitmask( bitmask, tokenizerInfo.getVocabSize() ); const curToken = encodedTokens[i]; const accepted = matcher.acceptToken(curToken); if (!accepted) { throw Error("Expect token to be accepted"); } } // 9. Clean up matcher.dispose(); compiledGrammar.dispose(); compiler.dispose(); tokenizerInfo.dispose(); } async function testAll() { await jsonExample(); await jsonSchemaExample(); await structuralTagExample(); } testAll(); xgrammar-0.2.3/web/example/tsconfig.json000066400000000000000000000001711521764210300202510ustar00rootroot00000000000000{ "compilerOptions": { }, "include": ["src"], "exclude": ["node_modules", "build", "dist", "rollup.config.js"] } xgrammar-0.2.3/web/jest.config.cjs000066400000000000000000000006041521764210300170220ustar00rootroot00000000000000module.exports = { preset: "ts-jest/presets/default-esm", testEnvironment: "node", extensionsToTreatAsEsm: [".ts"], transform: { "^.+\\.(ts|tsx)$": ["ts-jest", { useESM: true, tsconfig: "tsconfig.json" }], }, moduleNameMapper: { "^(\\.{1,2}/.*)\\.js$": "$1", "^@mlc-ai/web-tokenizers$": "/tests/web_tokenizers_shim.mjs", }, }; xgrammar-0.2.3/web/package-lock.json000066400000000000000000007050731521764210300173400ustar00rootroot00000000000000{ "name": "@mlc-ai/web-xgrammar", "version": "0.1.27", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@mlc-ai/web-xgrammar", "version": "0.1.27", "license": "Apache-2.0", "devDependencies": { "@eslint/eslintrc": "^3.3.1", "@eslint/js": "^9.9.0", "@jest/globals": "^30.2.0", "@mlc-ai/web-tokenizers": "^0.1.6", "@rollup/plugin-commonjs": "^29.0.0", "@rollup/plugin-node-resolve": "^16.0.3", "@rollup/plugin-wasm": "^6.2.2", "@types/jest": "^30.0.0", "@typescript-eslint/eslint-plugin": "8.48.0", "@typescript-eslint/parser": "8.48.0", "eslint": "^9.39.1", "jest": "^30.2.0", "rollup": "^4.53.3", "rollup-plugin-typescript2": "^0.36.0", "ts-jest": "^29.4.5", "tslib": "^2.3.1", "typescript": "^5.9.3", "typescript-eslint": "^8.47.0" } }, "node_modules/@babel/code-frame": { "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-validator-identifier": "^7.27.1", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/compat-data": { "version": "7.28.5", "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.28.5.tgz", "integrity": "sha512-6uFXyCayocRbqhZOB+6XcuZbkMNimwfVGFji8CTZnCzOHVGvDqzvitu1re2AU5LROliz7eQPhB8CpAMvnx9EjA==", "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/core": { "version": "7.28.5", "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.5.tgz", "integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==", "dev": true, "license": "MIT", "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.5", "@babel/helper-compilation-targets": "^7.27.2", "@babel/helper-module-transforms": "^7.28.3", "@babel/helpers": "^7.28.4", "@babel/parser": "^7.28.5", "@babel/template": "^7.27.2", "@babel/traverse": "^7.28.5", "@babel/types": "^7.28.5", "@jridgewell/remapping": "^2.3.5", "convert-source-map": "^2.0.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", "json5": "^2.2.3", "semver": "^6.3.1" }, "engines": { "node": ">=6.9.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/babel" } }, "node_modules/@babel/core/node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true, "license": "ISC", "bin": { "semver": "bin/semver.js" } }, "node_modules/@babel/generator": { "version": "7.28.5", "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.28.5.tgz", "integrity": "sha512-3EwLFhZ38J4VyIP6WNtt2kUdW9dokXA9Cr4IVIFHuCpZ3H8/YFOl5JjZHisrn1fATPBmKKqXzDFvh9fUwHz6CQ==", "dev": true, "license": "MIT", "dependencies": { "@babel/parser": "^7.28.5", "@babel/types": "^7.28.5", "@jridgewell/gen-mapping": "^0.3.12", "@jridgewell/trace-mapping": "^0.3.28", "jsesc": "^3.0.2" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-compilation-targets": { "version": "7.27.2", "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.27.2.tgz", "integrity": "sha512-2+1thGUUWWjLTYTHZWK1n8Yga0ijBz1XAhUXcKy81rd5g6yh7hGqMp45v7cadSbEHc9G3OTv45SyneRN3ps4DQ==", "dev": true, "license": "MIT", "dependencies": { "@babel/compat-data": "^7.27.2", "@babel/helper-validator-option": "^7.27.1", "browserslist": "^4.24.0", "lru-cache": "^5.1.1", "semver": "^6.3.1" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-compilation-targets/node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true, "license": "ISC", "bin": { "semver": "bin/semver.js" } }, "node_modules/@babel/helper-globals": { "version": "7.28.0", "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-module-imports": { "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.27.1.tgz", "integrity": "sha512-0gSFWUPNXNopqtIPQvlD5WgXYI5GY2kP2cCvoT8kczjbfcfuIljTbcWrulD1CIPIX2gt1wghbDy08yE1p+/r3w==", "dev": true, "license": "MIT", "dependencies": { "@babel/traverse": "^7.27.1", "@babel/types": "^7.27.1" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-module-transforms": { "version": "7.28.3", "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.3.tgz", "integrity": "sha512-gytXUbs8k2sXS9PnQptz5o0QnpLL51SwASIORY6XaBKF88nsOT0Zw9szLqlSGQDP/4TljBAD5y98p2U1fqkdsw==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-module-imports": "^7.27.1", "@babel/helper-validator-identifier": "^7.27.1", "@babel/traverse": "^7.28.3" }, "engines": { "node": ">=6.9.0" }, "peerDependencies": { "@babel/core": "^7.0.0" } }, "node_modules/@babel/helper-plugin-utils": { "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.27.1.tgz", "integrity": "sha512-1gn1Up5YXka3YYAHGKpbideQ5Yjf1tDa9qYcgysz+cNCXukyLl6DjPXhD3VRwSb8c0J9tA4b2+rHEZtc6R0tlw==", "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-string-parser": { "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-validator-identifier": { "version": "7.28.5", "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-validator-option": { "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helpers": { "version": "7.28.4", "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.28.4.tgz", "integrity": "sha512-HFN59MmQXGHVyYadKLVumYsA9dBFun/ldYxipEjzA4196jpLZd8UjEEBLkbEkvfYreDqJhZxYAWFPtrfhNpj4w==", "dev": true, "license": "MIT", "dependencies": { "@babel/template": "^7.27.2", "@babel/types": "^7.28.4" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/parser": { "version": "7.28.5", "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.5.tgz", "integrity": "sha512-KKBU1VGYR7ORr3At5HAtUQ+TV3SzRCXmA/8OdDZiLDBIZxVyzXuztPjfLd3BV1PRAQGCMWWSHYhL0F8d5uHBDQ==", "dev": true, "license": "MIT", "dependencies": { "@babel/types": "^7.28.5" }, "bin": { "parser": "bin/babel-parser.js" }, "engines": { "node": ">=6.0.0" } }, "node_modules/@babel/plugin-syntax-async-generators": { "version": "7.8.4", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-bigint": { "version": "7.8.3", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz", "integrity": "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-class-properties": { "version": "7.12.13", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-plugin-utils": "^7.12.13" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-class-static-block": { "version": "7.14.5", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz", "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-plugin-utils": "^7.14.5" }, "engines": { "node": ">=6.9.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-import-attributes": { "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.27.1.tgz", "integrity": "sha512-oFT0FrKHgF53f4vOsZGi2Hh3I35PfSmVs4IBFLFj4dnafP+hIWDLg3VyKmUHfLoLHlyxY4C7DGtmHuJgn+IGww==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-import-meta": { "version": "7.10.4", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz", "integrity": "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-plugin-utils": "^7.10.4" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-json-strings": { "version": "7.8.3", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-jsx": { "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.27.1.tgz", "integrity": "sha512-y8YTNIeKoyhGd9O0Jiyzyyqk8gdjnumGTQPsz0xOZOQ2RmkVJeZ1vmmfIvFEKqucBG6axJGBZDE/7iI5suUI/w==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-logical-assignment-operators": { "version": "7.10.4", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-plugin-utils": "^7.10.4" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": { "version": "7.8.3", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-numeric-separator": { "version": "7.10.4", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-plugin-utils": "^7.10.4" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-object-rest-spread": { "version": "7.8.3", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-optional-catch-binding": { "version": "7.8.3", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-optional-chaining": { "version": "7.8.3", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-private-property-in-object": { "version": "7.14.5", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz", "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-plugin-utils": "^7.14.5" }, "engines": { "node": ">=6.9.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-top-level-await": { "version": "7.14.5", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-plugin-utils": "^7.14.5" }, "engines": { "node": ">=6.9.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-typescript": { "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.27.1.tgz", "integrity": "sha512-xfYCBMxveHrRMnAWl1ZlPXOZjzkN82THFvLhQhFXFt81Z5HnN+EtUkZhv/zcKpmT3fzmWZB0ywiBrbC3vogbwQ==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-plugin-utils": "^7.27.1" }, "engines": { "node": ">=6.9.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/template": { "version": "7.27.2", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.27.2.tgz", "integrity": "sha512-LPDZ85aEJyYSd18/DkjNh4/y1ntkE5KwUHWTiqgRxruuZL2F1yuHligVHLvcHY2vMHXttKFpJn6LwfI7cw7ODw==", "dev": true, "license": "MIT", "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/parser": "^7.27.2", "@babel/types": "^7.27.1" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/traverse": { "version": "7.28.5", "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.28.5.tgz", "integrity": "sha512-TCCj4t55U90khlYkVV/0TfkJkAkUg3jZFA3Neb7unZT8CPok7iiRfaX0F+WnqWqt7OxhOn0uBKXCw4lbL8W0aQ==", "dev": true, "license": "MIT", "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.5", "@babel/helper-globals": "^7.28.0", "@babel/parser": "^7.28.5", "@babel/template": "^7.27.2", "@babel/types": "^7.28.5", "debug": "^4.3.1" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/types": { "version": "7.28.5", "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.28.5.tgz", "integrity": "sha512-qQ5m48eI/MFLQ5PxQj4PFaprjyCTLI37ElWMmNs0K8Lk3dVeOdNpB3ks8jc7yM5CDmVC73eMVk/trk3fgmrUpA==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-string-parser": "^7.27.1", "@babel/helper-validator-identifier": "^7.28.5" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@bcoe/v8-coverage": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true, "license": "MIT" }, "node_modules/@emnapi/core": { "version": "1.7.1", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.7.1.tgz", "integrity": "sha512-o1uhUASyo921r2XtHYOHy7gdkGLge8ghBEQHMWmyJFoXlpU58kIrhhN3w26lpQb6dspetweapMn2CSNwQ8I4wg==", "dev": true, "license": "MIT", "optional": true, "dependencies": { "@emnapi/wasi-threads": "1.1.0", "tslib": "^2.4.0" } }, "node_modules/@emnapi/runtime": { "version": "1.7.1", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.7.1.tgz", "integrity": "sha512-PVtJr5CmLwYAU9PZDMITZoR5iAOShYREoR45EyyLrbntV50mdePTgUn4AmOw90Ifcj+x2kRjdzr1HP3RrNiHGA==", "dev": true, "license": "MIT", "optional": true, "dependencies": { "tslib": "^2.4.0" } }, "node_modules/@emnapi/wasi-threads": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.1.0.tgz", "integrity": "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==", "dev": true, "license": "MIT", "optional": true, "dependencies": { "tslib": "^2.4.0" } }, "node_modules/@eslint-community/eslint-utils": { "version": "4.9.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.0.tgz", "integrity": "sha512-ayVFHdtZ+hsq1t2Dy24wCmGXGe4q9Gu3smhLYALJrr473ZH27MsnSL+LKUlimp4BWJqMDMLmPpx/Q9R3OAlL4g==", "dev": true, "license": "MIT", "dependencies": { "eslint-visitor-keys": "^3.4.3" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, "funding": { "url": "https://opencollective.com/eslint" }, "peerDependencies": { "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" } }, "node_modules/@eslint-community/regexpp": { "version": "4.12.2", "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", "dev": true, "license": "MIT", "engines": { "node": "^12.0.0 || ^14.0.0 || >=16.0.0" } }, "node_modules/@eslint/config-array": { "version": "0.21.1", "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.1.tgz", "integrity": "sha512-aw1gNayWpdI/jSYVgzN5pL0cfzU02GT3NBpeT/DXbx1/1x7ZKxFPd9bwrzygx/qiwIQiJ1sw/zD8qY/kRvlGHA==", "dev": true, "license": "Apache-2.0", "dependencies": { "@eslint/object-schema": "^2.1.7", "debug": "^4.3.1", "minimatch": "^3.1.2" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, "node_modules/@eslint/config-helpers": { "version": "0.4.2", "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz", "integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==", "dev": true, "license": "Apache-2.0", "dependencies": { "@eslint/core": "^0.17.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, "node_modules/@eslint/core": { "version": "0.17.0", "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz", "integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==", "dev": true, "license": "Apache-2.0", "dependencies": { "@types/json-schema": "^7.0.15" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, "node_modules/@eslint/eslintrc": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.1.tgz", "integrity": "sha512-gtF186CXhIl1p4pJNGZw8Yc6RlshoePRvE0X91oPGb3vZ8pM3qOS9W9NGPat9LziaBV7XrJWGylNQXkGcnM3IQ==", "dev": true, "license": "MIT", "dependencies": { "ajv": "^6.12.4", "debug": "^4.3.2", "espree": "^10.0.1", "globals": "^14.0.0", "ignore": "^5.2.0", "import-fresh": "^3.2.1", "js-yaml": "^4.1.0", "minimatch": "^3.1.2", "strip-json-comments": "^3.1.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "url": "https://opencollective.com/eslint" } }, "node_modules/@eslint/js": { "version": "9.39.1", "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.1.tgz", "integrity": "sha512-S26Stp4zCy88tH94QbBv3XCuzRQiZ9yXofEILmglYTh/Ug/a9/umqvgFtYBAo3Lp0nsI/5/qH1CCrbdK3AP1Tw==", "dev": true, "license": "MIT", "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "url": "https://eslint.org/donate" } }, "node_modules/@eslint/object-schema": { "version": "2.1.7", "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.7.tgz", "integrity": "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==", "dev": true, "license": "Apache-2.0", "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, "node_modules/@eslint/plugin-kit": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz", "integrity": "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==", "dev": true, "license": "Apache-2.0", "dependencies": { "@eslint/core": "^0.17.0", "levn": "^0.4.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, "node_modules/@humanfs/core": { "version": "0.19.1", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", "integrity": "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==", "dev": true, "license": "Apache-2.0", "engines": { "node": ">=18.18.0" } }, "node_modules/@humanfs/node": { "version": "0.16.7", "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.7.tgz", "integrity": "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==", "dev": true, "license": "Apache-2.0", "dependencies": { "@humanfs/core": "^0.19.1", "@humanwhocodes/retry": "^0.4.0" }, "engines": { "node": ">=18.18.0" } }, "node_modules/@humanwhocodes/module-importer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", "dev": true, "license": "Apache-2.0", "engines": { "node": ">=12.22" }, "funding": { "type": "github", "url": "https://github.com/sponsors/nzakas" } }, "node_modules/@humanwhocodes/retry": { "version": "0.4.3", "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", "dev": true, "license": "Apache-2.0", "engines": { "node": ">=18.18" }, "funding": { "type": "github", "url": "https://github.com/sponsors/nzakas" } }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", "dev": true, "license": "ISC", "dependencies": { "string-width": "^5.1.2", "string-width-cjs": "npm:string-width@^4.2.0", "strip-ansi": "^7.0.1", "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", "wrap-ansi": "^8.1.0", "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" }, "engines": { "node": ">=12" } }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==", "dev": true, "license": "ISC", "dependencies": { "camelcase": "^5.3.1", "find-up": "^4.1.0", "get-package-type": "^0.1.0", "js-yaml": "^3.13.1", "resolve-from": "^5.0.0" }, "engines": { "node": ">=8" } }, "node_modules/@istanbuljs/load-nyc-config/node_modules/argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", "dev": true, "license": "MIT", "dependencies": { "sprintf-js": "~1.0.2" } }, "node_modules/@istanbuljs/load-nyc-config/node_modules/find-up": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", "dev": true, "license": "MIT", "dependencies": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" }, "engines": { "node": ">=8" } }, "node_modules/@istanbuljs/load-nyc-config/node_modules/js-yaml": { "version": "3.14.2", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", "dev": true, "license": "MIT", "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "node_modules/@istanbuljs/load-nyc-config/node_modules/locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", "dev": true, "license": "MIT", "dependencies": { "p-locate": "^4.1.0" }, "engines": { "node": ">=8" } }, "node_modules/@istanbuljs/load-nyc-config/node_modules/p-limit": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", "dev": true, "license": "MIT", "dependencies": { "p-try": "^2.0.0" }, "engines": { "node": ">=6" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/@istanbuljs/load-nyc-config/node_modules/p-locate": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", "dev": true, "license": "MIT", "dependencies": { "p-limit": "^2.2.0" }, "engines": { "node": ">=8" } }, "node_modules/@istanbuljs/load-nyc-config/node_modules/resolve-from": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", "dev": true, "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/@istanbuljs/schema": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", "dev": true, "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/@jest/console": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/@jest/console/-/console-30.2.0.tgz", "integrity": "sha512-+O1ifRjkvYIkBqASKWgLxrpEhQAAE7hY77ALLUufSk5717KfOShg6IbqLmdsLMPdUiFvA2kTs0R7YZy+l0IzZQ==", "dev": true, "license": "MIT", "dependencies": { "@jest/types": "30.2.0", "@types/node": "*", "chalk": "^4.1.2", "jest-message-util": "30.2.0", "jest-util": "30.2.0", "slash": "^3.0.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/core": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/@jest/core/-/core-30.2.0.tgz", "integrity": "sha512-03W6IhuhjqTlpzh/ojut/pDB2LPRygyWX8ExpgHtQA8H/3K7+1vKmcINx5UzeOX1se6YEsBsOHQ1CRzf3fOwTQ==", "dev": true, "license": "MIT", "dependencies": { "@jest/console": "30.2.0", "@jest/pattern": "30.0.1", "@jest/reporters": "30.2.0", "@jest/test-result": "30.2.0", "@jest/transform": "30.2.0", "@jest/types": "30.2.0", "@types/node": "*", "ansi-escapes": "^4.3.2", "chalk": "^4.1.2", "ci-info": "^4.2.0", "exit-x": "^0.2.2", "graceful-fs": "^4.2.11", "jest-changed-files": "30.2.0", "jest-config": "30.2.0", "jest-haste-map": "30.2.0", "jest-message-util": "30.2.0", "jest-regex-util": "30.0.1", "jest-resolve": "30.2.0", "jest-resolve-dependencies": "30.2.0", "jest-runner": "30.2.0", "jest-runtime": "30.2.0", "jest-snapshot": "30.2.0", "jest-util": "30.2.0", "jest-validate": "30.2.0", "jest-watcher": "30.2.0", "micromatch": "^4.0.8", "pretty-format": "30.2.0", "slash": "^3.0.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" }, "peerDependencies": { "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" }, "peerDependenciesMeta": { "node-notifier": { "optional": true } } }, "node_modules/@jest/diff-sequences": { "version": "30.0.1", "resolved": "https://registry.npmjs.org/@jest/diff-sequences/-/diff-sequences-30.0.1.tgz", "integrity": "sha512-n5H8QLDJ47QqbCNn5SuFjCRDrOLEZ0h8vAHCK5RL9Ls7Xa8AQLa/YxAc9UjFqoEDM48muwtBGjtMY5cr0PLDCw==", "dev": true, "license": "MIT", "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/environment": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-30.2.0.tgz", "integrity": "sha512-/QPTL7OBJQ5ac09UDRa3EQes4gt1FTEG/8jZ/4v5IVzx+Cv7dLxlVIvfvSVRiiX2drWyXeBjkMSR8hvOWSog5g==", "dev": true, "license": "MIT", "dependencies": { "@jest/fake-timers": "30.2.0", "@jest/types": "30.2.0", "@types/node": "*", "jest-mock": "30.2.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/expect": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-30.2.0.tgz", "integrity": "sha512-V9yxQK5erfzx99Sf+7LbhBwNWEZ9eZay8qQ9+JSC0TrMR1pMDHLMY+BnVPacWU6Jamrh252/IKo4F1Xn/zfiqA==", "dev": true, "license": "MIT", "dependencies": { "expect": "30.2.0", "jest-snapshot": "30.2.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/expect-utils": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-30.2.0.tgz", "integrity": "sha512-1JnRfhqpD8HGpOmQp180Fo9Zt69zNtC+9lR+kT7NVL05tNXIi+QC8Csz7lfidMoVLPD3FnOtcmp0CEFnxExGEA==", "dev": true, "license": "MIT", "dependencies": { "@jest/get-type": "30.1.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/fake-timers": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-30.2.0.tgz", "integrity": "sha512-HI3tRLjRxAbBy0VO8dqqm7Hb2mIa8d5bg/NJkyQcOk7V118ObQML8RC5luTF/Zsg4474a+gDvhce7eTnP4GhYw==", "dev": true, "license": "MIT", "dependencies": { "@jest/types": "30.2.0", "@sinonjs/fake-timers": "^13.0.0", "@types/node": "*", "jest-message-util": "30.2.0", "jest-mock": "30.2.0", "jest-util": "30.2.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/get-type": { "version": "30.1.0", "resolved": "https://registry.npmjs.org/@jest/get-type/-/get-type-30.1.0.tgz", "integrity": "sha512-eMbZE2hUnx1WV0pmURZY9XoXPkUYjpc55mb0CrhtdWLtzMQPFvu/rZkTLZFTsdaVQa+Tr4eWAteqcUzoawq/uA==", "dev": true, "license": "MIT", "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/globals": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-30.2.0.tgz", "integrity": "sha512-b63wmnKPaK+6ZZfpYhz9K61oybvbI1aMcIs80++JI1O1rR1vaxHUCNqo3ITu6NU0d4V34yZFoHMn/uoKr/Rwfw==", "dev": true, "license": "MIT", "dependencies": { "@jest/environment": "30.2.0", "@jest/expect": "30.2.0", "@jest/types": "30.2.0", "jest-mock": "30.2.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/pattern": { "version": "30.0.1", "resolved": "https://registry.npmjs.org/@jest/pattern/-/pattern-30.0.1.tgz", "integrity": "sha512-gWp7NfQW27LaBQz3TITS8L7ZCQ0TLvtmI//4OwlQRx4rnWxcPNIYjxZpDcN4+UlGxgm3jS5QPz8IPTCkb59wZA==", "dev": true, "license": "MIT", "dependencies": { "@types/node": "*", "jest-regex-util": "30.0.1" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/reporters": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-30.2.0.tgz", "integrity": "sha512-DRyW6baWPqKMa9CzeiBjHwjd8XeAyco2Vt8XbcLFjiwCOEKOvy82GJ8QQnJE9ofsxCMPjH4MfH8fCWIHHDKpAQ==", "dev": true, "license": "MIT", "dependencies": { "@bcoe/v8-coverage": "^0.2.3", "@jest/console": "30.2.0", "@jest/test-result": "30.2.0", "@jest/transform": "30.2.0", "@jest/types": "30.2.0", "@jridgewell/trace-mapping": "^0.3.25", "@types/node": "*", "chalk": "^4.1.2", "collect-v8-coverage": "^1.0.2", "exit-x": "^0.2.2", "glob": "^10.3.10", "graceful-fs": "^4.2.11", "istanbul-lib-coverage": "^3.0.0", "istanbul-lib-instrument": "^6.0.0", "istanbul-lib-report": "^3.0.0", "istanbul-lib-source-maps": "^5.0.0", "istanbul-reports": "^3.1.3", "jest-message-util": "30.2.0", "jest-util": "30.2.0", "jest-worker": "30.2.0", "slash": "^3.0.0", "string-length": "^4.0.2", "v8-to-istanbul": "^9.0.1" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" }, "peerDependencies": { "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" }, "peerDependenciesMeta": { "node-notifier": { "optional": true } } }, "node_modules/@jest/schemas": { "version": "30.0.5", "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.5.tgz", "integrity": "sha512-DmdYgtezMkh3cpU8/1uyXakv3tJRcmcXxBOcO0tbaozPwpmh4YMsnWrQm9ZmZMfa5ocbxzbFk6O4bDPEc/iAnA==", "dev": true, "license": "MIT", "dependencies": { "@sinclair/typebox": "^0.34.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/snapshot-utils": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/@jest/snapshot-utils/-/snapshot-utils-30.2.0.tgz", "integrity": "sha512-0aVxM3RH6DaiLcjj/b0KrIBZhSX1373Xci4l3cW5xiUWPctZ59zQ7jj4rqcJQ/Z8JuN/4wX3FpJSa3RssVvCug==", "dev": true, "license": "MIT", "dependencies": { "@jest/types": "30.2.0", "chalk": "^4.1.2", "graceful-fs": "^4.2.11", "natural-compare": "^1.4.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/source-map": { "version": "30.0.1", "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-30.0.1.tgz", "integrity": "sha512-MIRWMUUR3sdbP36oyNyhbThLHyJ2eEDClPCiHVbrYAe5g3CHRArIVpBw7cdSB5fr+ofSfIb2Tnsw8iEHL0PYQg==", "dev": true, "license": "MIT", "dependencies": { "@jridgewell/trace-mapping": "^0.3.25", "callsites": "^3.1.0", "graceful-fs": "^4.2.11" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/test-result": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-30.2.0.tgz", "integrity": "sha512-RF+Z+0CCHkARz5HT9mcQCBulb1wgCP3FBvl9VFokMX27acKphwyQsNuWH3c+ojd1LeWBLoTYoxF0zm6S/66mjg==", "dev": true, "license": "MIT", "dependencies": { "@jest/console": "30.2.0", "@jest/types": "30.2.0", "@types/istanbul-lib-coverage": "^2.0.6", "collect-v8-coverage": "^1.0.2" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/test-sequencer": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-30.2.0.tgz", "integrity": "sha512-wXKgU/lk8fKXMu/l5Hog1R61bL4q5GCdT6OJvdAFz1P+QrpoFuLU68eoKuVc4RbrTtNnTL5FByhWdLgOPSph+Q==", "dev": true, "license": "MIT", "dependencies": { "@jest/test-result": "30.2.0", "graceful-fs": "^4.2.11", "jest-haste-map": "30.2.0", "slash": "^3.0.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/transform": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-30.2.0.tgz", "integrity": "sha512-XsauDV82o5qXbhalKxD7p4TZYYdwcaEXC77PPD2HixEFF+6YGppjrAAQurTl2ECWcEomHBMMNS9AH3kcCFx8jA==", "dev": true, "license": "MIT", "dependencies": { "@babel/core": "^7.27.4", "@jest/types": "30.2.0", "@jridgewell/trace-mapping": "^0.3.25", "babel-plugin-istanbul": "^7.0.1", "chalk": "^4.1.2", "convert-source-map": "^2.0.0", "fast-json-stable-stringify": "^2.1.0", "graceful-fs": "^4.2.11", "jest-haste-map": "30.2.0", "jest-regex-util": "30.0.1", "jest-util": "30.2.0", "micromatch": "^4.0.8", "pirates": "^4.0.7", "slash": "^3.0.0", "write-file-atomic": "^5.0.1" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/types": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.2.0.tgz", "integrity": "sha512-H9xg1/sfVvyfU7o3zMfBEjQ1gcsdeTMgqHoYdN79tuLqfTtuu7WckRA1R5whDwOzxaZAeMKTYWqP+WCAi0CHsg==", "dev": true, "license": "MIT", "dependencies": { "@jest/pattern": "30.0.1", "@jest/schemas": "30.0.5", "@types/istanbul-lib-coverage": "^2.0.6", "@types/istanbul-reports": "^3.0.4", "@types/node": "*", "@types/yargs": "^17.0.33", "chalk": "^4.1.2" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", "dev": true, "license": "MIT", "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "node_modules/@jridgewell/remapping": { "version": "2.3.5", "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", "dev": true, "license": "MIT", "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" } }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", "dev": true, "license": "MIT", "engines": { "node": ">=6.0.0" } }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.5.5", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", "dev": true, "license": "MIT" }, "node_modules/@jridgewell/trace-mapping": { "version": "0.3.31", "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", "dev": true, "license": "MIT", "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "node_modules/@mlc-ai/web-tokenizers": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/@mlc-ai/web-tokenizers/-/web-tokenizers-0.1.6.tgz", "integrity": "sha512-A5GSqUSnMjDkPoXBFFtbbW3F/qygCixuwbi7/EUMzcpgwFOAhD9vSZZBchK3IpC0c6TKlcDqAYDYp8vpb8/4vA==", "dev": true, "license": "Apache-2.0" }, "node_modules/@napi-rs/wasm-runtime": { "version": "0.2.12", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz", "integrity": "sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==", "dev": true, "license": "MIT", "optional": true, "dependencies": { "@emnapi/core": "^1.4.3", "@emnapi/runtime": "^1.4.3", "@tybys/wasm-util": "^0.10.0" } }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", "dev": true, "license": "MIT", "optional": true, "engines": { "node": ">=14" } }, "node_modules/@pkgr/core": { "version": "0.2.9", "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.2.9.tgz", "integrity": "sha512-QNqXyfVS2wm9hweSYD2O7F0G06uurj9kZ96TRQE5Y9hU7+tgdZwIkbAKc5Ocy1HxEY2kuDQa6cQ1WRs/O5LFKA==", "dev": true, "license": "MIT", "engines": { "node": "^12.20.0 || ^14.18.0 || >=16.0.0" }, "funding": { "url": "https://opencollective.com/pkgr" } }, "node_modules/@rollup/plugin-commonjs": { "version": "29.0.0", "resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-29.0.0.tgz", "integrity": "sha512-U2YHaxR2cU/yAiwKJtJRhnyLk7cifnQw0zUpISsocBDoHDJn+HTV74ABqnwr5bEgWUwFZC9oFL6wLe21lHu5eQ==", "dev": true, "license": "MIT", "dependencies": { "@rollup/pluginutils": "^5.0.1", "commondir": "^1.0.1", "estree-walker": "^2.0.2", "fdir": "^6.2.0", "is-reference": "1.2.1", "magic-string": "^0.30.3", "picomatch": "^4.0.2" }, "engines": { "node": ">=16.0.0 || 14 >= 14.17" }, "peerDependencies": { "rollup": "^2.68.0||^3.0.0||^4.0.0" }, "peerDependenciesMeta": { "rollup": { "optional": true } } }, "node_modules/@rollup/plugin-node-resolve": { "version": "16.0.3", "resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-16.0.3.tgz", "integrity": "sha512-lUYM3UBGuM93CnMPG1YocWu7X802BrNF3jW2zny5gQyLQgRFJhV1Sq0Zi74+dh/6NBx1DxFC4b4GXg9wUCG5Qg==", "dev": true, "license": "MIT", "dependencies": { "@rollup/pluginutils": "^5.0.1", "@types/resolve": "1.20.2", "deepmerge": "^4.2.2", "is-module": "^1.0.0", "resolve": "^1.22.1" }, "engines": { "node": ">=14.0.0" }, "peerDependencies": { "rollup": "^2.78.0||^3.0.0||^4.0.0" }, "peerDependenciesMeta": { "rollup": { "optional": true } } }, "node_modules/@rollup/plugin-wasm": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/@rollup/plugin-wasm/-/plugin-wasm-6.2.2.tgz", "integrity": "sha512-gpC4R1G9Ni92ZIRTexqbhX7U+9estZrbhP+9SRb0DW9xpB9g7j34r+J2hqrcW/lRI7dJaU84MxZM0Rt82tqYPQ==", "dev": true, "license": "MIT", "dependencies": { "@rollup/pluginutils": "^5.0.2" }, "engines": { "node": ">=14.0.0" }, "peerDependencies": { "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" }, "peerDependenciesMeta": { "rollup": { "optional": true } } }, "node_modules/@rollup/pluginutils": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.3.0.tgz", "integrity": "sha512-5EdhGZtnu3V88ces7s53hhfK5KSASnJZv8Lulpc04cWO3REESroJXg73DFsOmgbU2BhwV0E20bu2IDZb3VKW4Q==", "dev": true, "license": "MIT", "dependencies": { "@types/estree": "^1.0.0", "estree-walker": "^2.0.2", "picomatch": "^4.0.2" }, "engines": { "node": ">=14.0.0" }, "peerDependencies": { "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" }, "peerDependenciesMeta": { "rollup": { "optional": true } } }, "node_modules/@rollup/rollup-android-arm-eabi": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.53.3.tgz", "integrity": "sha512-mRSi+4cBjrRLoaal2PnqH82Wqyb+d3HsPUN/W+WslCXsZsyHa9ZeQQX/pQsZaVIWDkPcpV6jJ+3KLbTbgnwv8w==", "cpu": [ "arm" ], "dev": true, "license": "MIT", "optional": true, "os": [ "android" ] }, "node_modules/@rollup/rollup-android-arm64": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.53.3.tgz", "integrity": "sha512-CbDGaMpdE9sh7sCmTrTUyllhrg65t6SwhjlMJsLr+J8YjFuPmCEjbBSx4Z/e4SmDyH3aB5hGaJUP2ltV/vcs4w==", "cpu": [ "arm64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "android" ] }, "node_modules/@rollup/rollup-darwin-arm64": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.53.3.tgz", "integrity": "sha512-Nr7SlQeqIBpOV6BHHGZgYBuSdanCXuw09hon14MGOLGmXAFYjx1wNvquVPmpZnl0tLjg25dEdr4IQ6GgyToCUA==", "cpu": [ "arm64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "darwin" ] }, "node_modules/@rollup/rollup-darwin-x64": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.53.3.tgz", "integrity": "sha512-DZ8N4CSNfl965CmPktJ8oBnfYr3F8dTTNBQkRlffnUarJ2ohudQD17sZBa097J8xhQ26AwhHJ5mvUyQW8ddTsQ==", "cpu": [ "x64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "darwin" ] }, "node_modules/@rollup/rollup-freebsd-arm64": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.53.3.tgz", "integrity": "sha512-yMTrCrK92aGyi7GuDNtGn2sNW+Gdb4vErx4t3Gv/Tr+1zRb8ax4z8GWVRfr3Jw8zJWvpGHNpss3vVlbF58DZ4w==", "cpu": [ "arm64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "freebsd" ] }, "node_modules/@rollup/rollup-freebsd-x64": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.53.3.tgz", "integrity": "sha512-lMfF8X7QhdQzseM6XaX0vbno2m3hlyZFhwcndRMw8fbAGUGL3WFMBdK0hbUBIUYcEcMhVLr1SIamDeuLBnXS+Q==", "cpu": [ "x64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "freebsd" ] }, "node_modules/@rollup/rollup-linux-arm-gnueabihf": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.53.3.tgz", "integrity": "sha512-k9oD15soC/Ln6d2Wv/JOFPzZXIAIFLp6B+i14KhxAfnq76ajt0EhYc5YPeX6W1xJkAdItcVT+JhKl1QZh44/qw==", "cpu": [ "arm" ], "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-arm-musleabihf": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.53.3.tgz", "integrity": "sha512-vTNlKq+N6CK/8UktsrFuc+/7NlEYVxgaEgRXVUVK258Z5ymho29skzW1sutgYjqNnquGwVUObAaxae8rZ6YMhg==", "cpu": [ "arm" ], "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-arm64-gnu": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.53.3.tgz", "integrity": "sha512-RGrFLWgMhSxRs/EWJMIFM1O5Mzuz3Xy3/mnxJp/5cVhZ2XoCAxJnmNsEyeMJtpK+wu0FJFWz+QF4mjCA7AUQ3w==", "cpu": [ "arm64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-arm64-musl": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.53.3.tgz", "integrity": "sha512-kASyvfBEWYPEwe0Qv4nfu6pNkITLTb32p4yTgzFCocHnJLAHs+9LjUu9ONIhvfT/5lv4YS5muBHyuV84epBo/A==", "cpu": [ "arm64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-loong64-gnu": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.53.3.tgz", "integrity": "sha512-JiuKcp2teLJwQ7vkJ95EwESWkNRFJD7TQgYmCnrPtlu50b4XvT5MOmurWNrCj3IFdyjBQ5p9vnrX4JM6I8OE7g==", "cpu": [ "loong64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-ppc64-gnu": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.53.3.tgz", "integrity": "sha512-EoGSa8nd6d3T7zLuqdojxC20oBfNT8nexBbB/rkxgKj5T5vhpAQKKnD+h3UkoMuTyXkP5jTjK/ccNRmQrPNDuw==", "cpu": [ "ppc64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-riscv64-gnu": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.53.3.tgz", "integrity": "sha512-4s+Wped2IHXHPnAEbIB0YWBv7SDohqxobiiPA1FIWZpX+w9o2i4LezzH/NkFUl8LRci/8udci6cLq+jJQlh+0g==", "cpu": [ "riscv64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-riscv64-musl": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.53.3.tgz", "integrity": "sha512-68k2g7+0vs2u9CxDt5ktXTngsxOQkSEV/xBbwlqYcUrAVh6P9EgMZvFsnHy4SEiUl46Xf0IObWVbMvPrr2gw8A==", "cpu": [ "riscv64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-s390x-gnu": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.53.3.tgz", "integrity": "sha512-VYsFMpULAz87ZW6BVYw3I6sWesGpsP9OPcyKe8ofdg9LHxSbRMd7zrVrr5xi/3kMZtpWL/wC+UIJWJYVX5uTKg==", "cpu": [ "s390x" ], "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-x64-gnu": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.53.3.tgz", "integrity": "sha512-3EhFi1FU6YL8HTUJZ51imGJWEX//ajQPfqWLI3BQq4TlvHy4X0MOr5q3D2Zof/ka0d5FNdPwZXm3Yyib/UEd+w==", "cpu": [ "x64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-x64-musl": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.53.3.tgz", "integrity": "sha512-eoROhjcc6HbZCJr+tvVT8X4fW3/5g/WkGvvmwz/88sDtSJzO7r/blvoBDgISDiCjDRZmHpwud7h+6Q9JxFwq1Q==", "cpu": [ "x64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-openharmony-arm64": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.53.3.tgz", "integrity": "sha512-OueLAWgrNSPGAdUdIjSWXw+u/02BRTcnfw9PN41D2vq/JSEPnJnVuBgw18VkN8wcd4fjUs+jFHVM4t9+kBSNLw==", "cpu": [ "arm64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "openharmony" ] }, "node_modules/@rollup/rollup-win32-arm64-msvc": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.53.3.tgz", "integrity": "sha512-GOFuKpsxR/whszbF/bzydebLiXIHSgsEUp6M0JI8dWvi+fFa1TD6YQa4aSZHtpmh2/uAlj/Dy+nmby3TJ3pkTw==", "cpu": [ "arm64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "win32" ] }, "node_modules/@rollup/rollup-win32-ia32-msvc": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.53.3.tgz", "integrity": "sha512-iah+THLcBJdpfZ1TstDFbKNznlzoxa8fmnFYK4V67HvmuNYkVdAywJSoteUszvBQ9/HqN2+9AZghbajMsFT+oA==", "cpu": [ "ia32" ], "dev": true, "license": "MIT", "optional": true, "os": [ "win32" ] }, "node_modules/@rollup/rollup-win32-x64-gnu": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.53.3.tgz", "integrity": "sha512-J9QDiOIZlZLdcot5NXEepDkstocktoVjkaKUtqzgzpt2yWjGlbYiKyp05rWwk4nypbYUNoFAztEgixoLaSETkg==", "cpu": [ "x64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "win32" ] }, "node_modules/@rollup/rollup-win32-x64-msvc": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.53.3.tgz", "integrity": "sha512-UhTd8u31dXadv0MopwGgNOBpUVROFKWVQgAg5N1ESyCz8AuBcMqm4AuTjrwgQKGDfoFuz02EuMRHQIw/frmYKQ==", "cpu": [ "x64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "win32" ] }, "node_modules/@sinclair/typebox": { "version": "0.34.41", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.41.tgz", "integrity": "sha512-6gS8pZzSXdyRHTIqoqSVknxolr1kzfy4/CeDnrzsVz8TTIWUbOBr6gnzOmTYJ3eXQNh4IYHIGi5aIL7sOZ2G/g==", "dev": true, "license": "MIT" }, "node_modules/@sinonjs/commons": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", "dev": true, "license": "BSD-3-Clause", "dependencies": { "type-detect": "4.0.8" } }, "node_modules/@sinonjs/fake-timers": { "version": "13.0.5", "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-13.0.5.tgz", "integrity": "sha512-36/hTbH2uaWuGVERyC6da9YwGWnzUZXuPro/F2LfsdOsLnCojz/iSH8MxUt/FD2S5XBSVPhmArFUXcpCQ2Hkiw==", "dev": true, "license": "BSD-3-Clause", "dependencies": { "@sinonjs/commons": "^3.0.1" } }, "node_modules/@tybys/wasm-util": { "version": "0.10.1", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", "integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==", "dev": true, "license": "MIT", "optional": true, "dependencies": { "tslib": "^2.4.0" } }, "node_modules/@types/babel__core": { "version": "7.20.5", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", "dev": true, "license": "MIT", "dependencies": { "@babel/parser": "^7.20.7", "@babel/types": "^7.20.7", "@types/babel__generator": "*", "@types/babel__template": "*", "@types/babel__traverse": "*" } }, "node_modules/@types/babel__generator": { "version": "7.27.0", "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", "dev": true, "license": "MIT", "dependencies": { "@babel/types": "^7.0.0" } }, "node_modules/@types/babel__template": { "version": "7.4.4", "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", "dev": true, "license": "MIT", "dependencies": { "@babel/parser": "^7.1.0", "@babel/types": "^7.0.0" } }, "node_modules/@types/babel__traverse": { "version": "7.28.0", "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", "dev": true, "license": "MIT", "dependencies": { "@babel/types": "^7.28.2" } }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", "dev": true, "license": "MIT" }, "node_modules/@types/istanbul-lib-coverage": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", "dev": true, "license": "MIT" }, "node_modules/@types/istanbul-lib-report": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.3.tgz", "integrity": "sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==", "dev": true, "license": "MIT", "dependencies": { "@types/istanbul-lib-coverage": "*" } }, "node_modules/@types/istanbul-reports": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.4.tgz", "integrity": "sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==", "dev": true, "license": "MIT", "dependencies": { "@types/istanbul-lib-report": "*" } }, "node_modules/@types/jest": { "version": "30.0.0", "resolved": "https://registry.npmjs.org/@types/jest/-/jest-30.0.0.tgz", "integrity": "sha512-XTYugzhuwqWjws0CVz8QpM36+T+Dz5mTEBKhNs/esGLnCIlGdRy+Dq78NRjd7ls7r8BC8ZRMOrKlkO1hU0JOwA==", "dev": true, "license": "MIT", "dependencies": { "expect": "^30.0.0", "pretty-format": "^30.0.0" } }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", "dev": true, "license": "MIT" }, "node_modules/@types/node": { "version": "24.10.1", "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", "dev": true, "license": "MIT", "dependencies": { "undici-types": "~7.16.0" } }, "node_modules/@types/resolve": { "version": "1.20.2", "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.20.2.tgz", "integrity": "sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q==", "dev": true, "license": "MIT" }, "node_modules/@types/stack-utils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", "dev": true, "license": "MIT" }, "node_modules/@types/yargs": { "version": "17.0.35", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.35.tgz", "integrity": "sha512-qUHkeCyQFxMXg79wQfTtfndEC+N9ZZg76HJftDJp+qH2tV7Gj4OJi7l+PiWwJ+pWtW8GwSmqsDj/oymhrTWXjg==", "dev": true, "license": "MIT", "dependencies": { "@types/yargs-parser": "*" } }, "node_modules/@types/yargs-parser": { "version": "21.0.3", "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz", "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", "dev": true, "license": "MIT" }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "8.48.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.48.0.tgz", "integrity": "sha512-XxXP5tL1txl13YFtrECECQYeZjBZad4fyd3cFV4a19LkAY/bIp9fev3US4S5fDVV2JaYFiKAZ/GRTOLer+mbyQ==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.10.0", "@typescript-eslint/scope-manager": "8.48.0", "@typescript-eslint/type-utils": "8.48.0", "@typescript-eslint/utils": "8.48.0", "@typescript-eslint/visitor-keys": "8.48.0", "graphemer": "^1.4.0", "ignore": "^7.0.0", "natural-compare": "^1.4.0", "ts-api-utils": "^2.1.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { "@typescript-eslint/parser": "^8.48.0", "eslint": "^8.57.0 || ^9.0.0", "typescript": ">=4.8.4 <6.0.0" } }, "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { "version": "7.0.5", "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", "dev": true, "license": "MIT", "engines": { "node": ">= 4" } }, "node_modules/@typescript-eslint/parser": { "version": "8.48.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.48.0.tgz", "integrity": "sha512-jCzKdm/QK0Kg4V4IK/oMlRZlY+QOcdjv89U2NgKHZk1CYTj82/RVSx1mV/0gqCVMJ/DA+Zf/S4NBWNF8GQ+eqQ==", "dev": true, "license": "MIT", "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.48.0", "@typescript-eslint/types": "8.48.0", "@typescript-eslint/typescript-estree": "8.48.0", "@typescript-eslint/visitor-keys": "8.48.0", "debug": "^4.3.4" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0", "typescript": ">=4.8.4 <6.0.0" } }, "node_modules/@typescript-eslint/project-service": { "version": "8.48.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.48.0.tgz", "integrity": "sha512-Ne4CTZyRh1BecBf84siv42wv5vQvVmgtk8AuiEffKTUo3DrBaGYZueJSxxBZ8fjk/N3DrgChH4TOdIOwOwiqqw==", "dev": true, "license": "MIT", "dependencies": { "@typescript-eslint/tsconfig-utils": "^8.48.0", "@typescript-eslint/types": "^8.48.0", "debug": "^4.3.4" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { "typescript": ">=4.8.4 <6.0.0" } }, "node_modules/@typescript-eslint/scope-manager": { "version": "8.48.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.48.0.tgz", "integrity": "sha512-uGSSsbrtJrLduti0Q1Q9+BF1/iFKaxGoQwjWOIVNJv0o6omrdyR8ct37m4xIl5Zzpkp69Kkmvom7QFTtue89YQ==", "dev": true, "license": "MIT", "dependencies": { "@typescript-eslint/types": "8.48.0", "@typescript-eslint/visitor-keys": "8.48.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" } }, "node_modules/@typescript-eslint/tsconfig-utils": { "version": "8.48.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.48.0.tgz", "integrity": "sha512-WNebjBdFdyu10sR1M4OXTt2OkMd5KWIL+LLfeH9KhgP+jzfDV/LI3eXzwJ1s9+Yc0Kzo2fQCdY/OpdusCMmh6w==", "dev": true, "license": "MIT", "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { "typescript": ">=4.8.4 <6.0.0" } }, "node_modules/@typescript-eslint/type-utils": { "version": "8.48.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.48.0.tgz", "integrity": "sha512-zbeVaVqeXhhab6QNEKfK96Xyc7UQuoFWERhEnj3mLVnUWrQnv15cJNseUni7f3g557gm0e46LZ6IJ4NJVOgOpw==", "dev": true, "license": "MIT", "dependencies": { "@typescript-eslint/types": "8.48.0", "@typescript-eslint/typescript-estree": "8.48.0", "@typescript-eslint/utils": "8.48.0", "debug": "^4.3.4", "ts-api-utils": "^2.1.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0", "typescript": ">=4.8.4 <6.0.0" } }, "node_modules/@typescript-eslint/types": { "version": "8.48.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.48.0.tgz", "integrity": "sha512-cQMcGQQH7kwKoVswD1xdOytxQR60MWKM1di26xSUtxehaDs/32Zpqsu5WJlXTtTTqyAVK8R7hvsUnIXRS+bjvA==", "dev": true, "license": "MIT", "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" } }, "node_modules/@typescript-eslint/typescript-estree": { "version": "8.48.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.48.0.tgz", "integrity": "sha512-ljHab1CSO4rGrQIAyizUS6UGHHCiAYhbfcIZ1zVJr5nMryxlXMVWS3duFPSKvSUbFPwkXMFk1k0EMIjub4sRRQ==", "dev": true, "license": "MIT", "dependencies": { "@typescript-eslint/project-service": "8.48.0", "@typescript-eslint/tsconfig-utils": "8.48.0", "@typescript-eslint/types": "8.48.0", "@typescript-eslint/visitor-keys": "8.48.0", "debug": "^4.3.4", "minimatch": "^9.0.4", "semver": "^7.6.0", "tinyglobby": "^0.2.15", "ts-api-utils": "^2.1.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { "typescript": ">=4.8.4 <6.0.0" } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^2.0.1" }, "engines": { "node": ">=16 || 14 >=14.17" }, "funding": { "url": "https://github.com/sponsors/isaacs" } }, "node_modules/@typescript-eslint/utils": { "version": "8.48.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.48.0.tgz", "integrity": "sha512-yTJO1XuGxCsSfIVt1+1UrLHtue8xz16V8apzPYI06W0HbEbEWHxHXgZaAgavIkoh+GeV6hKKd5jm0sS6OYxWXQ==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.7.0", "@typescript-eslint/scope-manager": "8.48.0", "@typescript-eslint/types": "8.48.0", "@typescript-eslint/typescript-estree": "8.48.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0", "typescript": ">=4.8.4 <6.0.0" } }, "node_modules/@typescript-eslint/visitor-keys": { "version": "8.48.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.48.0.tgz", "integrity": "sha512-T0XJMaRPOH3+LBbAfzR2jalckP1MSG/L9eUtY0DEzUyVaXJ/t6zN0nR7co5kz0Jko/nkSYCBRkz1djvjajVTTg==", "dev": true, "license": "MIT", "dependencies": { "@typescript-eslint/types": "8.48.0", "eslint-visitor-keys": "^4.2.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" } }, "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", "dev": true, "license": "Apache-2.0", "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "url": "https://opencollective.com/eslint" } }, "node_modules/@ungap/structured-clone": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", "integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==", "dev": true, "license": "ISC" }, "node_modules/@unrs/resolver-binding-android-arm-eabi": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-android-arm-eabi/-/resolver-binding-android-arm-eabi-1.11.1.tgz", "integrity": "sha512-ppLRUgHVaGRWUx0R0Ut06Mjo9gBaBkg3v/8AxusGLhsIotbBLuRk51rAzqLC8gq6NyyAojEXglNjzf6R948DNw==", "cpu": [ "arm" ], "dev": true, "license": "MIT", "optional": true, "os": [ "android" ] }, "node_modules/@unrs/resolver-binding-android-arm64": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-android-arm64/-/resolver-binding-android-arm64-1.11.1.tgz", "integrity": "sha512-lCxkVtb4wp1v+EoN+HjIG9cIIzPkX5OtM03pQYkG+U5O/wL53LC4QbIeazgiKqluGeVEeBlZahHalCaBvU1a2g==", "cpu": [ "arm64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "android" ] }, "node_modules/@unrs/resolver-binding-darwin-arm64": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-darwin-arm64/-/resolver-binding-darwin-arm64-1.11.1.tgz", "integrity": "sha512-gPVA1UjRu1Y/IsB/dQEsp2V1pm44Of6+LWvbLc9SDk1c2KhhDRDBUkQCYVWe6f26uJb3fOK8saWMgtX8IrMk3g==", "cpu": [ "arm64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "darwin" ] }, "node_modules/@unrs/resolver-binding-darwin-x64": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-darwin-x64/-/resolver-binding-darwin-x64-1.11.1.tgz", "integrity": "sha512-cFzP7rWKd3lZaCsDze07QX1SC24lO8mPty9vdP+YVa3MGdVgPmFc59317b2ioXtgCMKGiCLxJ4HQs62oz6GfRQ==", "cpu": [ "x64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "darwin" ] }, "node_modules/@unrs/resolver-binding-freebsd-x64": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-freebsd-x64/-/resolver-binding-freebsd-x64-1.11.1.tgz", "integrity": "sha512-fqtGgak3zX4DCB6PFpsH5+Kmt/8CIi4Bry4rb1ho6Av2QHTREM+47y282Uqiu3ZRF5IQioJQ5qWRV6jduA+iGw==", "cpu": [ "x64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "freebsd" ] }, "node_modules/@unrs/resolver-binding-linux-arm-gnueabihf": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm-gnueabihf/-/resolver-binding-linux-arm-gnueabihf-1.11.1.tgz", "integrity": "sha512-u92mvlcYtp9MRKmP+ZvMmtPN34+/3lMHlyMj7wXJDeXxuM0Vgzz0+PPJNsro1m3IZPYChIkn944wW8TYgGKFHw==", "cpu": [ "arm" ], "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@unrs/resolver-binding-linux-arm-musleabihf": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm-musleabihf/-/resolver-binding-linux-arm-musleabihf-1.11.1.tgz", "integrity": "sha512-cINaoY2z7LVCrfHkIcmvj7osTOtm6VVT16b5oQdS4beibX2SYBwgYLmqhBjA1t51CarSaBuX5YNsWLjsqfW5Cw==", "cpu": [ "arm" ], "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@unrs/resolver-binding-linux-arm64-gnu": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm64-gnu/-/resolver-binding-linux-arm64-gnu-1.11.1.tgz", "integrity": "sha512-34gw7PjDGB9JgePJEmhEqBhWvCiiWCuXsL9hYphDF7crW7UgI05gyBAi6MF58uGcMOiOqSJ2ybEeCvHcq0BCmQ==", "cpu": [ "arm64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@unrs/resolver-binding-linux-arm64-musl": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm64-musl/-/resolver-binding-linux-arm64-musl-1.11.1.tgz", "integrity": "sha512-RyMIx6Uf53hhOtJDIamSbTskA99sPHS96wxVE/bJtePJJtpdKGXO1wY90oRdXuYOGOTuqjT8ACccMc4K6QmT3w==", "cpu": [ "arm64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@unrs/resolver-binding-linux-ppc64-gnu": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-ppc64-gnu/-/resolver-binding-linux-ppc64-gnu-1.11.1.tgz", "integrity": "sha512-D8Vae74A4/a+mZH0FbOkFJL9DSK2R6TFPC9M+jCWYia/q2einCubX10pecpDiTmkJVUH+y8K3BZClycD8nCShA==", "cpu": [ "ppc64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@unrs/resolver-binding-linux-riscv64-gnu": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-riscv64-gnu/-/resolver-binding-linux-riscv64-gnu-1.11.1.tgz", "integrity": "sha512-frxL4OrzOWVVsOc96+V3aqTIQl1O2TjgExV4EKgRY09AJ9leZpEg8Ak9phadbuX0BA4k8U5qtvMSQQGGmaJqcQ==", "cpu": [ "riscv64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@unrs/resolver-binding-linux-riscv64-musl": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-riscv64-musl/-/resolver-binding-linux-riscv64-musl-1.11.1.tgz", "integrity": "sha512-mJ5vuDaIZ+l/acv01sHoXfpnyrNKOk/3aDoEdLO/Xtn9HuZlDD6jKxHlkN8ZhWyLJsRBxfv9GYM2utQ1SChKew==", "cpu": [ "riscv64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@unrs/resolver-binding-linux-s390x-gnu": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-s390x-gnu/-/resolver-binding-linux-s390x-gnu-1.11.1.tgz", "integrity": "sha512-kELo8ebBVtb9sA7rMe1Cph4QHreByhaZ2QEADd9NzIQsYNQpt9UkM9iqr2lhGr5afh885d/cB5QeTXSbZHTYPg==", "cpu": [ "s390x" ], "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@unrs/resolver-binding-linux-x64-gnu": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-x64-gnu/-/resolver-binding-linux-x64-gnu-1.11.1.tgz", "integrity": "sha512-C3ZAHugKgovV5YvAMsxhq0gtXuwESUKc5MhEtjBpLoHPLYM+iuwSj3lflFwK3DPm68660rZ7G8BMcwSro7hD5w==", "cpu": [ "x64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@unrs/resolver-binding-linux-x64-musl": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-x64-musl/-/resolver-binding-linux-x64-musl-1.11.1.tgz", "integrity": "sha512-rV0YSoyhK2nZ4vEswT/QwqzqQXw5I6CjoaYMOX0TqBlWhojUf8P94mvI7nuJTeaCkkds3QE4+zS8Ko+GdXuZtA==", "cpu": [ "x64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@unrs/resolver-binding-wasm32-wasi": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-wasm32-wasi/-/resolver-binding-wasm32-wasi-1.11.1.tgz", "integrity": "sha512-5u4RkfxJm+Ng7IWgkzi3qrFOvLvQYnPBmjmZQ8+szTK/b31fQCnleNl1GgEt7nIsZRIf5PLhPwT0WM+q45x/UQ==", "cpu": [ "wasm32" ], "dev": true, "license": "MIT", "optional": true, "dependencies": { "@napi-rs/wasm-runtime": "^0.2.11" }, "engines": { "node": ">=14.0.0" } }, "node_modules/@unrs/resolver-binding-win32-arm64-msvc": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-arm64-msvc/-/resolver-binding-win32-arm64-msvc-1.11.1.tgz", "integrity": "sha512-nRcz5Il4ln0kMhfL8S3hLkxI85BXs3o8EYoattsJNdsX4YUU89iOkVn7g0VHSRxFuVMdM4Q1jEpIId1Ihim/Uw==", "cpu": [ "arm64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "win32" ] }, "node_modules/@unrs/resolver-binding-win32-ia32-msvc": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-ia32-msvc/-/resolver-binding-win32-ia32-msvc-1.11.1.tgz", "integrity": "sha512-DCEI6t5i1NmAZp6pFonpD5m7i6aFrpofcp4LA2i8IIq60Jyo28hamKBxNrZcyOwVOZkgsRp9O2sXWBWP8MnvIQ==", "cpu": [ "ia32" ], "dev": true, "license": "MIT", "optional": true, "os": [ "win32" ] }, "node_modules/@unrs/resolver-binding-win32-x64-msvc": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-x64-msvc/-/resolver-binding-win32-x64-msvc-1.11.1.tgz", "integrity": "sha512-lrW200hZdbfRtztbygyaq/6jP6AKE8qQN2KvPcJ+x7wiD038YtnYtZ82IMNJ69GJibV7bwL3y9FgK+5w/pYt6g==", "cpu": [ "x64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "win32" ] }, "node_modules/acorn": { "version": "8.15.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", "peer": true, "bin": { "acorn": "bin/acorn" }, "engines": { "node": ">=0.4.0" } }, "node_modules/acorn-jsx": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", "dev": true, "license": "MIT", "peerDependencies": { "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "dev": true, "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" }, "funding": { "type": "github", "url": "https://github.com/sponsors/epoberezkin" } }, "node_modules/ansi-escapes": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", "dev": true, "license": "MIT", "dependencies": { "type-fest": "^0.21.3" }, "engines": { "node": ">=8" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/ansi-regex": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", "dev": true, "license": "MIT", "engines": { "node": ">=12" }, "funding": { "url": "https://github.com/chalk/ansi-regex?sponsor=1" } }, "node_modules/ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", "dev": true, "license": "MIT", "dependencies": { "color-convert": "^2.0.1" }, "engines": { "node": ">=8" }, "funding": { "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, "node_modules/anymatch": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", "dev": true, "license": "ISC", "dependencies": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" }, "engines": { "node": ">= 8" } }, "node_modules/anymatch/node_modules/picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", "dev": true, "license": "MIT", "engines": { "node": ">=8.6" }, "funding": { "url": "https://github.com/sponsors/jonschlinkert" } }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true, "license": "Python-2.0" }, "node_modules/babel-jest": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-30.2.0.tgz", "integrity": "sha512-0YiBEOxWqKkSQWL9nNGGEgndoeL0ZpWrbLMNL5u/Kaxrli3Eaxlt3ZtIDktEvXt4L/R9r3ODr2zKwGM/2BjxVw==", "dev": true, "license": "MIT", "dependencies": { "@jest/transform": "30.2.0", "@types/babel__core": "^7.20.5", "babel-plugin-istanbul": "^7.0.1", "babel-preset-jest": "30.2.0", "chalk": "^4.1.2", "graceful-fs": "^4.2.11", "slash": "^3.0.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" }, "peerDependencies": { "@babel/core": "^7.11.0 || ^8.0.0-0" } }, "node_modules/babel-plugin-istanbul": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-7.0.1.tgz", "integrity": "sha512-D8Z6Qm8jCvVXtIRkBnqNHX0zJ37rQcFJ9u8WOS6tkYOsRdHBzypCstaxWiu5ZIlqQtviRYbgnRLSoCEvjqcqbA==", "dev": true, "license": "BSD-3-Clause", "workspaces": [ "test/babel-8" ], "dependencies": { "@babel/helper-plugin-utils": "^7.0.0", "@istanbuljs/load-nyc-config": "^1.0.0", "@istanbuljs/schema": "^0.1.3", "istanbul-lib-instrument": "^6.0.2", "test-exclude": "^6.0.0" }, "engines": { "node": ">=12" } }, "node_modules/babel-plugin-jest-hoist": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-30.2.0.tgz", "integrity": "sha512-ftzhzSGMUnOzcCXd6WHdBGMyuwy15Wnn0iyyWGKgBDLxf9/s5ABuraCSpBX2uG0jUg4rqJnxsLc5+oYBqoxVaA==", "dev": true, "license": "MIT", "dependencies": { "@types/babel__core": "^7.20.5" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/babel-preset-current-node-syntax": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.2.0.tgz", "integrity": "sha512-E/VlAEzRrsLEb2+dv8yp3bo4scof3l9nR4lrld+Iy5NyVqgVYUJnDAmunkhPMisRI32Qc4iRiz425d8vM++2fg==", "dev": true, "license": "MIT", "dependencies": { "@babel/plugin-syntax-async-generators": "^7.8.4", "@babel/plugin-syntax-bigint": "^7.8.3", "@babel/plugin-syntax-class-properties": "^7.12.13", "@babel/plugin-syntax-class-static-block": "^7.14.5", "@babel/plugin-syntax-import-attributes": "^7.24.7", "@babel/plugin-syntax-import-meta": "^7.10.4", "@babel/plugin-syntax-json-strings": "^7.8.3", "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", "@babel/plugin-syntax-numeric-separator": "^7.10.4", "@babel/plugin-syntax-object-rest-spread": "^7.8.3", "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", "@babel/plugin-syntax-optional-chaining": "^7.8.3", "@babel/plugin-syntax-private-property-in-object": "^7.14.5", "@babel/plugin-syntax-top-level-await": "^7.14.5" }, "peerDependencies": { "@babel/core": "^7.0.0 || ^8.0.0-0" } }, "node_modules/babel-preset-jest": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-30.2.0.tgz", "integrity": "sha512-US4Z3NOieAQumwFnYdUWKvUKh8+YSnS/gB3t6YBiz0bskpu7Pine8pPCheNxlPEW4wnUkma2a94YuW2q3guvCQ==", "dev": true, "license": "MIT", "dependencies": { "babel-plugin-jest-hoist": "30.2.0", "babel-preset-current-node-syntax": "^1.2.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" }, "peerDependencies": { "@babel/core": "^7.11.0 || ^8.0.0-beta.1" } }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true, "license": "MIT" }, "node_modules/baseline-browser-mapping": { "version": "2.8.31", "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.8.31.tgz", "integrity": "sha512-a28v2eWrrRWPpJSzxc+mKwm0ZtVx/G8SepdQZDArnXYU/XS+IF6mp8aB/4E+hH1tyGCoDo3KlUCdlSxGDsRkAw==", "dev": true, "license": "Apache-2.0", "bin": { "baseline-browser-mapping": "dist/cli.js" } }, "node_modules/brace-expansion": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "node_modules/braces": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, "license": "MIT", "dependencies": { "fill-range": "^7.1.1" }, "engines": { "node": ">=8" } }, "node_modules/browserslist": { "version": "4.28.0", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.0.tgz", "integrity": "sha512-tbydkR/CxfMwelN0vwdP/pLkDwyAASZ+VfWm4EOwlB6SWhx1sYnWLqo8N5j0rAzPfzfRaxt0mM/4wPU/Su84RQ==", "dev": true, "funding": [ { "type": "opencollective", "url": "https://opencollective.com/browserslist" }, { "type": "tidelift", "url": "https://tidelift.com/funding/github/npm/browserslist" }, { "type": "github", "url": "https://github.com/sponsors/ai" } ], "license": "MIT", "peer": true, "dependencies": { "baseline-browser-mapping": "^2.8.25", "caniuse-lite": "^1.0.30001754", "electron-to-chromium": "^1.5.249", "node-releases": "^2.0.27", "update-browserslist-db": "^1.1.4" }, "bin": { "browserslist": "cli.js" }, "engines": { "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, "node_modules/bs-logger": { "version": "0.2.6", "resolved": "https://registry.npmjs.org/bs-logger/-/bs-logger-0.2.6.tgz", "integrity": "sha512-pd8DCoxmbgc7hyPKOvxtqNcjYoOsABPQdcCUjGp3d42VR2CX1ORhk2A87oqqu5R1kk+76nsxZupkmyd+MVtCog==", "dev": true, "license": "MIT", "dependencies": { "fast-json-stable-stringify": "2.x" }, "engines": { "node": ">= 6" } }, "node_modules/bser": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz", "integrity": "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==", "dev": true, "license": "Apache-2.0", "dependencies": { "node-int64": "^0.4.0" } }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true, "license": "MIT" }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", "dev": true, "license": "MIT", "engines": { "node": ">=6" } }, "node_modules/camelcase": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", "dev": true, "license": "MIT", "engines": { "node": ">=6" } }, "node_modules/caniuse-lite": { "version": "1.0.30001757", "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001757.tgz", "integrity": "sha512-r0nnL/I28Zi/yjk1el6ilj27tKcdjLsNqAOZr0yVjWPrSQyHgKI2INaEWw21bAQSv2LXRt1XuCS/GomNpWOxsQ==", "dev": true, "funding": [ { "type": "opencollective", "url": "https://opencollective.com/browserslist" }, { "type": "tidelift", "url": "https://tidelift.com/funding/github/npm/caniuse-lite" }, { "type": "github", "url": "https://github.com/sponsors/ai" } ], "license": "CC-BY-4.0" }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" }, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/chalk/chalk?sponsor=1" } }, "node_modules/char-regex": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz", "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==", "dev": true, "license": "MIT", "engines": { "node": ">=10" } }, "node_modules/ci-info": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.3.1.tgz", "integrity": "sha512-Wdy2Igu8OcBpI2pZePZ5oWjPC38tmDVx5WKUXKwlLYkA0ozo85sLsLvkBbBn/sZaSCMFOGZJ14fvW9t5/d7kdA==", "dev": true, "funding": [ { "type": "github", "url": "https://github.com/sponsors/sibiraj-s" } ], "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/cjs-module-lexer": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-2.1.1.tgz", "integrity": "sha512-+CmxIZ/L2vNcEfvNtLdU0ZQ6mbq3FZnwAP2PPTiKP+1QOoKwlKlPgb8UKV0Dds7QVaMnHm+FwSft2VB0s/SLjQ==", "dev": true, "license": "MIT" }, "node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", "dev": true, "license": "ISC", "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", "wrap-ansi": "^7.0.0" }, "engines": { "node": ">=12" } }, "node_modules/cliui/node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true, "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/cliui/node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "dev": true, "license": "MIT" }, "node_modules/cliui/node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "dev": true, "license": "MIT", "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" }, "engines": { "node": ">=8" } }, "node_modules/cliui/node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" }, "engines": { "node": ">=8" } }, "node_modules/cliui/node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" }, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==", "dev": true, "license": "MIT", "engines": { "iojs": ">= 1.0.0", "node": ">= 0.12.0" } }, "node_modules/collect-v8-coverage": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.3.tgz", "integrity": "sha512-1L5aqIkwPfiodaMgQunkF1zRhNqifHBmtbbbxcr6yVxxBnliw4TDOW6NxpO8DJLgJ16OT+Y4ztZqP6p/FtXnAw==", "dev": true, "license": "MIT" }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", "dev": true, "license": "MIT", "dependencies": { "color-name": "~1.1.4" }, "engines": { "node": ">=7.0.0" } }, "node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true, "license": "MIT" }, "node_modules/commondir": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", "dev": true, "license": "MIT" }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "dev": true, "license": "MIT" }, "node_modules/convert-source-map": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", "dev": true, "license": "MIT" }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, "license": "MIT", "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" }, "engines": { "node": ">= 8" } }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "dev": true, "license": "MIT", "dependencies": { "ms": "^2.1.3" }, "engines": { "node": ">=6.0" }, "peerDependenciesMeta": { "supports-color": { "optional": true } } }, "node_modules/dedent": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.7.0.tgz", "integrity": "sha512-HGFtf8yhuhGhqO07SV79tRp+br4MnbdjeVxotpn1QBl30pcLLCQjX5b2295ll0fv8RKDKsmWYrl05usHM9CewQ==", "dev": true, "license": "MIT", "peerDependencies": { "babel-plugin-macros": "^3.1.0" }, "peerDependenciesMeta": { "babel-plugin-macros": { "optional": true } } }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", "dev": true, "license": "MIT" }, "node_modules/deepmerge": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" } }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==", "dev": true, "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", "dev": true, "license": "MIT" }, "node_modules/electron-to-chromium": { "version": "1.5.262", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.262.tgz", "integrity": "sha512-NlAsMteRHek05jRUxUR0a5jpjYq9ykk6+kO0yRaMi5moe7u0fVIOeQ3Y30A8dIiWFBNUoQGi1ljb1i5VtS9WQQ==", "dev": true, "license": "ISC" }, "node_modules/emittery": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", "integrity": "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==", "dev": true, "license": "MIT", "engines": { "node": ">=12" }, "funding": { "url": "https://github.com/sindresorhus/emittery?sponsor=1" } }, "node_modules/emoji-regex": { "version": "9.2.2", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", "dev": true, "license": "MIT" }, "node_modules/error-ex": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", "dev": true, "license": "MIT", "dependencies": { "is-arrayish": "^0.2.1" } }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", "dev": true, "license": "MIT", "engines": { "node": ">=6" } }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", "dev": true, "license": "MIT", "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/eslint": { "version": "9.39.1", "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.1.tgz", "integrity": "sha512-BhHmn2yNOFA9H9JmmIVKJmd288g9hrVRDkdoIgRCRuSySRUHH7r/DI6aAXW9T1WwUuY3DFgrcaqB+deURBLR5g==", "dev": true, "license": "MIT", "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", "@eslint/config-array": "^0.21.1", "@eslint/config-helpers": "^0.4.2", "@eslint/core": "^0.17.0", "@eslint/eslintrc": "^3.3.1", "@eslint/js": "9.39.1", "@eslint/plugin-kit": "^0.4.1", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.2", "@types/estree": "^1.0.6", "ajv": "^6.12.4", "chalk": "^4.0.0", "cross-spawn": "^7.0.6", "debug": "^4.3.2", "escape-string-regexp": "^4.0.0", "eslint-scope": "^8.4.0", "eslint-visitor-keys": "^4.2.1", "espree": "^10.4.0", "esquery": "^1.5.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", "file-entry-cache": "^8.0.0", "find-up": "^5.0.0", "glob-parent": "^6.0.2", "ignore": "^5.2.0", "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", "json-stable-stringify-without-jsonify": "^1.0.1", "lodash.merge": "^4.6.2", "minimatch": "^3.1.2", "natural-compare": "^1.4.0", "optionator": "^0.9.3" }, "bin": { "eslint": "bin/eslint.js" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "url": "https://eslint.org/donate" }, "peerDependencies": { "jiti": "*" }, "peerDependenciesMeta": { "jiti": { "optional": true } } }, "node_modules/eslint-scope": { "version": "8.4.0", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", "dev": true, "license": "BSD-2-Clause", "dependencies": { "esrecurse": "^4.3.0", "estraverse": "^5.2.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "url": "https://opencollective.com/eslint" } }, "node_modules/eslint-visitor-keys": { "version": "3.4.3", "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", "dev": true, "license": "Apache-2.0", "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, "funding": { "url": "https://opencollective.com/eslint" } }, "node_modules/eslint/node_modules/eslint-visitor-keys": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", "dev": true, "license": "Apache-2.0", "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "url": "https://opencollective.com/eslint" } }, "node_modules/espree": { "version": "10.4.0", "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", "dev": true, "license": "BSD-2-Clause", "dependencies": { "acorn": "^8.15.0", "acorn-jsx": "^5.3.2", "eslint-visitor-keys": "^4.2.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "url": "https://opencollective.com/eslint" } }, "node_modules/espree/node_modules/eslint-visitor-keys": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", "dev": true, "license": "Apache-2.0", "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "url": "https://opencollective.com/eslint" } }, "node_modules/esprima": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", "dev": true, "license": "BSD-2-Clause", "bin": { "esparse": "bin/esparse.js", "esvalidate": "bin/esvalidate.js" }, "engines": { "node": ">=4" } }, "node_modules/esquery": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz", "integrity": "sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg==", "dev": true, "license": "BSD-3-Clause", "dependencies": { "estraverse": "^5.1.0" }, "engines": { "node": ">=0.10" } }, "node_modules/esrecurse": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", "dev": true, "license": "BSD-2-Clause", "dependencies": { "estraverse": "^5.2.0" }, "engines": { "node": ">=4.0" } }, "node_modules/estraverse": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", "dev": true, "license": "BSD-2-Clause", "engines": { "node": ">=4.0" } }, "node_modules/estree-walker": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", "dev": true, "license": "MIT" }, "node_modules/esutils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", "dev": true, "license": "BSD-2-Clause", "engines": { "node": ">=0.10.0" } }, "node_modules/execa": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", "dev": true, "license": "MIT", "dependencies": { "cross-spawn": "^7.0.3", "get-stream": "^6.0.0", "human-signals": "^2.1.0", "is-stream": "^2.0.0", "merge-stream": "^2.0.0", "npm-run-path": "^4.0.1", "onetime": "^5.1.2", "signal-exit": "^3.0.3", "strip-final-newline": "^2.0.0" }, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sindresorhus/execa?sponsor=1" } }, "node_modules/execa/node_modules/signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", "dev": true, "license": "ISC" }, "node_modules/exit-x": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/exit-x/-/exit-x-0.2.2.tgz", "integrity": "sha512-+I6B/IkJc1o/2tiURyz/ivu/O0nKNEArIUB5O7zBrlDVJr22SCLH3xTeEry428LvFhRzIA1g8izguxJ/gbNcVQ==", "dev": true, "license": "MIT", "engines": { "node": ">= 0.8.0" } }, "node_modules/expect": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/expect/-/expect-30.2.0.tgz", "integrity": "sha512-u/feCi0GPsI+988gU2FLcsHyAHTU0MX1Wg68NhAnN7z/+C5wqG+CY8J53N9ioe8RXgaoz0nBR/TYMf3AycUuPw==", "dev": true, "license": "MIT", "dependencies": { "@jest/expect-utils": "30.2.0", "@jest/get-type": "30.1.0", "jest-matcher-utils": "30.2.0", "jest-message-util": "30.2.0", "jest-mock": "30.2.0", "jest-util": "30.2.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", "dev": true, "license": "MIT" }, "node_modules/fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", "dev": true, "license": "MIT" }, "node_modules/fast-levenshtein": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true, "license": "MIT" }, "node_modules/fb-watchman": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", "integrity": "sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==", "dev": true, "license": "Apache-2.0", "dependencies": { "bser": "2.1.1" } }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", "dev": true, "license": "MIT", "engines": { "node": ">=12.0.0" }, "peerDependencies": { "picomatch": "^3 || ^4" }, "peerDependenciesMeta": { "picomatch": { "optional": true } } }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", "dev": true, "license": "MIT", "dependencies": { "flat-cache": "^4.0.0" }, "engines": { "node": ">=16.0.0" } }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, "license": "MIT", "dependencies": { "to-regex-range": "^5.0.1" }, "engines": { "node": ">=8" } }, "node_modules/find-cache-dir": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.2.tgz", "integrity": "sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==", "dev": true, "license": "MIT", "dependencies": { "commondir": "^1.0.1", "make-dir": "^3.0.2", "pkg-dir": "^4.1.0" }, "engines": { "node": ">=8" }, "funding": { "url": "https://github.com/avajs/find-cache-dir?sponsor=1" } }, "node_modules/find-cache-dir/node_modules/make-dir": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", "dev": true, "license": "MIT", "dependencies": { "semver": "^6.0.0" }, "engines": { "node": ">=8" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/find-cache-dir/node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true, "license": "ISC", "bin": { "semver": "bin/semver.js" } }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", "dev": true, "license": "MIT", "dependencies": { "locate-path": "^6.0.0", "path-exists": "^4.0.0" }, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/flat-cache": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", "dev": true, "license": "MIT", "dependencies": { "flatted": "^3.2.9", "keyv": "^4.5.4" }, "engines": { "node": ">=16" } }, "node_modules/flatted": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", "integrity": "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==", "dev": true, "license": "ISC" }, "node_modules/foreground-child": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", "dev": true, "license": "ISC", "dependencies": { "cross-spawn": "^7.0.6", "signal-exit": "^4.0.1" }, "engines": { "node": ">=14" }, "funding": { "url": "https://github.com/sponsors/isaacs" } }, "node_modules/fs-extra": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", "dev": true, "license": "MIT", "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", "universalify": "^2.0.0" }, "engines": { "node": ">=12" } }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", "dev": true, "hasInstallScript": true, "license": "MIT", "optional": true, "os": [ "darwin" ], "engines": { "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" } }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" } }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", "dev": true, "license": "ISC", "engines": { "node": "6.* || 8.* || >= 10.*" } }, "node_modules/get-package-type": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", "dev": true, "license": "MIT", "engines": { "node": ">=8.0.0" } }, "node_modules/get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", "dev": true, "license": "MIT", "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/glob": { "version": "10.5.0", "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", "dev": true, "license": "ISC", "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", "minimatch": "^9.0.4", "minipass": "^7.1.2", "package-json-from-dist": "^1.0.0", "path-scurry": "^1.11.1" }, "bin": { "glob": "dist/esm/bin.mjs" }, "funding": { "url": "https://github.com/sponsors/isaacs" } }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", "dev": true, "license": "ISC", "dependencies": { "is-glob": "^4.0.3" }, "engines": { "node": ">=10.13.0" } }, "node_modules/glob/node_modules/brace-expansion": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" } }, "node_modules/glob/node_modules/minimatch": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^2.0.1" }, "engines": { "node": ">=16 || 14 >=14.17" }, "funding": { "url": "https://github.com/sponsors/isaacs" } }, "node_modules/globals": { "version": "14.0.0", "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", "dev": true, "license": "MIT", "engines": { "node": ">=18" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", "dev": true, "license": "ISC" }, "node_modules/graphemer": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", "dev": true, "license": "MIT" }, "node_modules/handlebars": { "version": "4.7.8", "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==", "dev": true, "license": "MIT", "dependencies": { "minimist": "^1.2.5", "neo-async": "^2.6.2", "source-map": "^0.6.1", "wordwrap": "^1.0.0" }, "bin": { "handlebars": "bin/handlebars" }, "engines": { "node": ">=0.4.7" }, "optionalDependencies": { "uglify-js": "^3.1.4" } }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", "dev": true, "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/hasown": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", "dev": true, "license": "MIT", "dependencies": { "function-bind": "^1.1.2" }, "engines": { "node": ">= 0.4" } }, "node_modules/html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", "dev": true, "license": "MIT" }, "node_modules/human-signals": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", "dev": true, "license": "Apache-2.0", "engines": { "node": ">=10.17.0" } }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", "dev": true, "license": "MIT", "engines": { "node": ">= 4" } }, "node_modules/import-fresh": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", "dev": true, "license": "MIT", "dependencies": { "parent-module": "^1.0.0", "resolve-from": "^4.0.0" }, "engines": { "node": ">=6" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/import-local": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz", "integrity": "sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==", "dev": true, "license": "MIT", "dependencies": { "pkg-dir": "^4.2.0", "resolve-cwd": "^3.0.0" }, "bin": { "import-local-fixture": "fixtures/cli.js" }, "engines": { "node": ">=8" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", "dev": true, "license": "MIT", "engines": { "node": ">=0.8.19" } }, "node_modules/is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", "dev": true, "license": "MIT" }, "node_modules/is-core-module": { "version": "2.16.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", "dev": true, "license": "MIT", "dependencies": { "hasown": "^2.0.2" }, "engines": { "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" } }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" } }, "node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", "dev": true, "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/is-generator-fn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz", "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", "dev": true, "license": "MIT", "engines": { "node": ">=6" } }, "node_modules/is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", "dev": true, "license": "MIT", "dependencies": { "is-extglob": "^2.1.1" }, "engines": { "node": ">=0.10.0" } }, "node_modules/is-module": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-module/-/is-module-1.0.0.tgz", "integrity": "sha512-51ypPSPCoTEIN9dy5Oy+h4pShgJmPCygKfyRCISBI+JoWT/2oJvK8QPxmwv7b/p239jXrm9M1mlQbyKJ5A152g==", "dev": true, "license": "MIT" }, "node_modules/is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", "dev": true, "license": "MIT", "engines": { "node": ">=0.12.0" } }, "node_modules/is-reference": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-1.2.1.tgz", "integrity": "sha512-U82MsXXiFIrjCK4otLT+o2NA2Cd2g5MLoOVXUZjIOhLurrRxpEXzI8O0KZHr3IjLvlAH1kTPYSuqer5T9ZVBKQ==", "dev": true, "license": "MIT", "dependencies": { "@types/estree": "*" } }, "node_modules/is-stream": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", "dev": true, "license": "MIT", "engines": { "node": ">=8" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "dev": true, "license": "ISC" }, "node_modules/istanbul-lib-coverage": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", "dev": true, "license": "BSD-3-Clause", "engines": { "node": ">=8" } }, "node_modules/istanbul-lib-instrument": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.3.tgz", "integrity": "sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q==", "dev": true, "license": "BSD-3-Clause", "dependencies": { "@babel/core": "^7.23.9", "@babel/parser": "^7.23.9", "@istanbuljs/schema": "^0.1.3", "istanbul-lib-coverage": "^3.2.0", "semver": "^7.5.4" }, "engines": { "node": ">=10" } }, "node_modules/istanbul-lib-report": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", "dev": true, "license": "BSD-3-Clause", "dependencies": { "istanbul-lib-coverage": "^3.0.0", "make-dir": "^4.0.0", "supports-color": "^7.1.0" }, "engines": { "node": ">=10" } }, "node_modules/istanbul-lib-source-maps": { "version": "5.0.6", "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-5.0.6.tgz", "integrity": "sha512-yg2d+Em4KizZC5niWhQaIomgf5WlL4vOOjZ5xGCmF8SnPE/mDWWXgvRExdcpCgh9lLRRa1/fSYp2ymmbJ1pI+A==", "dev": true, "license": "BSD-3-Clause", "dependencies": { "@jridgewell/trace-mapping": "^0.3.23", "debug": "^4.1.1", "istanbul-lib-coverage": "^3.0.0" }, "engines": { "node": ">=10" } }, "node_modules/istanbul-reports": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", "dev": true, "license": "BSD-3-Clause", "dependencies": { "html-escaper": "^2.0.0", "istanbul-lib-report": "^3.0.0" }, "engines": { "node": ">=8" } }, "node_modules/jackspeak": { "version": "3.4.3", "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "@isaacs/cliui": "^8.0.2" }, "funding": { "url": "https://github.com/sponsors/isaacs" }, "optionalDependencies": { "@pkgjs/parseargs": "^0.11.0" } }, "node_modules/jest": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest/-/jest-30.2.0.tgz", "integrity": "sha512-F26gjC0yWN8uAA5m5Ss8ZQf5nDHWGlN/xWZIh8S5SRbsEKBovwZhxGd6LJlbZYxBgCYOtreSUyb8hpXyGC5O4A==", "dev": true, "license": "MIT", "peer": true, "dependencies": { "@jest/core": "30.2.0", "@jest/types": "30.2.0", "import-local": "^3.2.0", "jest-cli": "30.2.0" }, "bin": { "jest": "bin/jest.js" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" }, "peerDependencies": { "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" }, "peerDependenciesMeta": { "node-notifier": { "optional": true } } }, "node_modules/jest-changed-files": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-30.2.0.tgz", "integrity": "sha512-L8lR1ChrRnSdfeOvTrwZMlnWV8G/LLjQ0nG9MBclwWZidA2N5FviRki0Bvh20WRMOX31/JYvzdqTJrk5oBdydQ==", "dev": true, "license": "MIT", "dependencies": { "execa": "^5.1.1", "jest-util": "30.2.0", "p-limit": "^3.1.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-circus": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-30.2.0.tgz", "integrity": "sha512-Fh0096NC3ZkFx05EP2OXCxJAREVxj1BcW/i6EWqqymcgYKWjyyDpral3fMxVcHXg6oZM7iULer9wGRFvfpl+Tg==", "dev": true, "license": "MIT", "dependencies": { "@jest/environment": "30.2.0", "@jest/expect": "30.2.0", "@jest/test-result": "30.2.0", "@jest/types": "30.2.0", "@types/node": "*", "chalk": "^4.1.2", "co": "^4.6.0", "dedent": "^1.6.0", "is-generator-fn": "^2.1.0", "jest-each": "30.2.0", "jest-matcher-utils": "30.2.0", "jest-message-util": "30.2.0", "jest-runtime": "30.2.0", "jest-snapshot": "30.2.0", "jest-util": "30.2.0", "p-limit": "^3.1.0", "pretty-format": "30.2.0", "pure-rand": "^7.0.0", "slash": "^3.0.0", "stack-utils": "^2.0.6" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-cli": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-30.2.0.tgz", "integrity": "sha512-Os9ukIvADX/A9sLt6Zse3+nmHtHaE6hqOsjQtNiugFTbKRHYIYtZXNGNK9NChseXy7djFPjndX1tL0sCTlfpAA==", "dev": true, "license": "MIT", "dependencies": { "@jest/core": "30.2.0", "@jest/test-result": "30.2.0", "@jest/types": "30.2.0", "chalk": "^4.1.2", "exit-x": "^0.2.2", "import-local": "^3.2.0", "jest-config": "30.2.0", "jest-util": "30.2.0", "jest-validate": "30.2.0", "yargs": "^17.7.2" }, "bin": { "jest": "bin/jest.js" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" }, "peerDependencies": { "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" }, "peerDependenciesMeta": { "node-notifier": { "optional": true } } }, "node_modules/jest-config": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-30.2.0.tgz", "integrity": "sha512-g4WkyzFQVWHtu6uqGmQR4CQxz/CH3yDSlhzXMWzNjDx843gYjReZnMRanjRCq5XZFuQrGDxgUaiYWE8BRfVckA==", "dev": true, "license": "MIT", "dependencies": { "@babel/core": "^7.27.4", "@jest/get-type": "30.1.0", "@jest/pattern": "30.0.1", "@jest/test-sequencer": "30.2.0", "@jest/types": "30.2.0", "babel-jest": "30.2.0", "chalk": "^4.1.2", "ci-info": "^4.2.0", "deepmerge": "^4.3.1", "glob": "^10.3.10", "graceful-fs": "^4.2.11", "jest-circus": "30.2.0", "jest-docblock": "30.2.0", "jest-environment-node": "30.2.0", "jest-regex-util": "30.0.1", "jest-resolve": "30.2.0", "jest-runner": "30.2.0", "jest-util": "30.2.0", "jest-validate": "30.2.0", "micromatch": "^4.0.8", "parse-json": "^5.2.0", "pretty-format": "30.2.0", "slash": "^3.0.0", "strip-json-comments": "^3.1.1" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" }, "peerDependencies": { "@types/node": "*", "esbuild-register": ">=3.4.0", "ts-node": ">=9.0.0" }, "peerDependenciesMeta": { "@types/node": { "optional": true }, "esbuild-register": { "optional": true }, "ts-node": { "optional": true } } }, "node_modules/jest-diff": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.2.0.tgz", "integrity": "sha512-dQHFo3Pt4/NLlG5z4PxZ/3yZTZ1C7s9hveiOj+GCN+uT109NC2QgsoVZsVOAvbJ3RgKkvyLGXZV9+piDpWbm6A==", "dev": true, "license": "MIT", "dependencies": { "@jest/diff-sequences": "30.0.1", "@jest/get-type": "30.1.0", "chalk": "^4.1.2", "pretty-format": "30.2.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-docblock": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-30.2.0.tgz", "integrity": "sha512-tR/FFgZKS1CXluOQzZvNH3+0z9jXr3ldGSD8bhyuxvlVUwbeLOGynkunvlTMxchC5urrKndYiwCFC0DLVjpOCA==", "dev": true, "license": "MIT", "dependencies": { "detect-newline": "^3.1.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-each": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-30.2.0.tgz", "integrity": "sha512-lpWlJlM7bCUf1mfmuqTA8+j2lNURW9eNafOy99knBM01i5CQeY5UH1vZjgT9071nDJac1M4XsbyI44oNOdhlDQ==", "dev": true, "license": "MIT", "dependencies": { "@jest/get-type": "30.1.0", "@jest/types": "30.2.0", "chalk": "^4.1.2", "jest-util": "30.2.0", "pretty-format": "30.2.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-environment-node": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.2.0.tgz", "integrity": "sha512-ElU8v92QJ9UrYsKrxDIKCxu6PfNj4Hdcktcn0JX12zqNdqWHB0N+hwOnnBBXvjLd2vApZtuLUGs1QSY+MsXoNA==", "dev": true, "license": "MIT", "dependencies": { "@jest/environment": "30.2.0", "@jest/fake-timers": "30.2.0", "@jest/types": "30.2.0", "@types/node": "*", "jest-mock": "30.2.0", "jest-util": "30.2.0", "jest-validate": "30.2.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-haste-map": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-30.2.0.tgz", "integrity": "sha512-sQA/jCb9kNt+neM0anSj6eZhLZUIhQgwDt7cPGjumgLM4rXsfb9kpnlacmvZz3Q5tb80nS+oG/if+NBKrHC+Xw==", "dev": true, "license": "MIT", "dependencies": { "@jest/types": "30.2.0", "@types/node": "*", "anymatch": "^3.1.3", "fb-watchman": "^2.0.2", "graceful-fs": "^4.2.11", "jest-regex-util": "30.0.1", "jest-util": "30.2.0", "jest-worker": "30.2.0", "micromatch": "^4.0.8", "walker": "^1.0.8" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" }, "optionalDependencies": { "fsevents": "^2.3.3" } }, "node_modules/jest-leak-detector": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-30.2.0.tgz", "integrity": "sha512-M6jKAjyzjHG0SrQgwhgZGy9hFazcudwCNovY/9HPIicmNSBuockPSedAP9vlPK6ONFJ1zfyH/M2/YYJxOz5cdQ==", "dev": true, "license": "MIT", "dependencies": { "@jest/get-type": "30.1.0", "pretty-format": "30.2.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-matcher-utils": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-30.2.0.tgz", "integrity": "sha512-dQ94Nq4dbzmUWkQ0ANAWS9tBRfqCrn0bV9AMYdOi/MHW726xn7eQmMeRTpX2ViC00bpNaWXq+7o4lIQ3AX13Hg==", "dev": true, "license": "MIT", "dependencies": { "@jest/get-type": "30.1.0", "chalk": "^4.1.2", "jest-diff": "30.2.0", "pretty-format": "30.2.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-message-util": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.2.0.tgz", "integrity": "sha512-y4DKFLZ2y6DxTWD4cDe07RglV88ZiNEdlRfGtqahfbIjfsw1nMCPx49Uev4IA/hWn3sDKyAnSPwoYSsAEdcimw==", "dev": true, "license": "MIT", "dependencies": { "@babel/code-frame": "^7.27.1", "@jest/types": "30.2.0", "@types/stack-utils": "^2.0.3", "chalk": "^4.1.2", "graceful-fs": "^4.2.11", "micromatch": "^4.0.8", "pretty-format": "30.2.0", "slash": "^3.0.0", "stack-utils": "^2.0.6" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-mock": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-30.2.0.tgz", "integrity": "sha512-JNNNl2rj4b5ICpmAcq+WbLH83XswjPbjH4T7yvGzfAGCPh1rw+xVNbtk+FnRslvt9lkCcdn9i1oAoKUuFsOxRw==", "dev": true, "license": "MIT", "dependencies": { "@jest/types": "30.2.0", "@types/node": "*", "jest-util": "30.2.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-pnp-resolver": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz", "integrity": "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==", "dev": true, "license": "MIT", "engines": { "node": ">=6" }, "peerDependencies": { "jest-resolve": "*" }, "peerDependenciesMeta": { "jest-resolve": { "optional": true } } }, "node_modules/jest-regex-util": { "version": "30.0.1", "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-30.0.1.tgz", "integrity": "sha512-jHEQgBXAgc+Gh4g0p3bCevgRCVRkB4VB70zhoAE48gxeSr1hfUOsM/C2WoJgVL7Eyg//hudYENbm3Ne+/dRVVA==", "dev": true, "license": "MIT", "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-resolve": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-30.2.0.tgz", "integrity": "sha512-TCrHSxPlx3tBY3hWNtRQKbtgLhsXa1WmbJEqBlTBrGafd5fiQFByy2GNCEoGR+Tns8d15GaL9cxEzKOO3GEb2A==", "dev": true, "license": "MIT", "dependencies": { "chalk": "^4.1.2", "graceful-fs": "^4.2.11", "jest-haste-map": "30.2.0", "jest-pnp-resolver": "^1.2.3", "jest-util": "30.2.0", "jest-validate": "30.2.0", "slash": "^3.0.0", "unrs-resolver": "^1.7.11" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-resolve-dependencies": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-30.2.0.tgz", "integrity": "sha512-xTOIGug/0RmIe3mmCqCT95yO0vj6JURrn1TKWlNbhiAefJRWINNPgwVkrVgt/YaerPzY3iItufd80v3lOrFJ2w==", "dev": true, "license": "MIT", "dependencies": { "jest-regex-util": "30.0.1", "jest-snapshot": "30.2.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-runner": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-30.2.0.tgz", "integrity": "sha512-PqvZ2B2XEyPEbclp+gV6KO/F1FIFSbIwewRgmROCMBo/aZ6J1w8Qypoj2pEOcg3G2HzLlaP6VUtvwCI8dM3oqQ==", "dev": true, "license": "MIT", "dependencies": { "@jest/console": "30.2.0", "@jest/environment": "30.2.0", "@jest/test-result": "30.2.0", "@jest/transform": "30.2.0", "@jest/types": "30.2.0", "@types/node": "*", "chalk": "^4.1.2", "emittery": "^0.13.1", "exit-x": "^0.2.2", "graceful-fs": "^4.2.11", "jest-docblock": "30.2.0", "jest-environment-node": "30.2.0", "jest-haste-map": "30.2.0", "jest-leak-detector": "30.2.0", "jest-message-util": "30.2.0", "jest-resolve": "30.2.0", "jest-runtime": "30.2.0", "jest-util": "30.2.0", "jest-watcher": "30.2.0", "jest-worker": "30.2.0", "p-limit": "^3.1.0", "source-map-support": "0.5.13" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-runtime": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-30.2.0.tgz", "integrity": "sha512-p1+GVX/PJqTucvsmERPMgCPvQJpFt4hFbM+VN3n8TMo47decMUcJbt+rgzwrEme0MQUA/R+1de2axftTHkKckg==", "dev": true, "license": "MIT", "dependencies": { "@jest/environment": "30.2.0", "@jest/fake-timers": "30.2.0", "@jest/globals": "30.2.0", "@jest/source-map": "30.0.1", "@jest/test-result": "30.2.0", "@jest/transform": "30.2.0", "@jest/types": "30.2.0", "@types/node": "*", "chalk": "^4.1.2", "cjs-module-lexer": "^2.1.0", "collect-v8-coverage": "^1.0.2", "glob": "^10.3.10", "graceful-fs": "^4.2.11", "jest-haste-map": "30.2.0", "jest-message-util": "30.2.0", "jest-mock": "30.2.0", "jest-regex-util": "30.0.1", "jest-resolve": "30.2.0", "jest-snapshot": "30.2.0", "jest-util": "30.2.0", "slash": "^3.0.0", "strip-bom": "^4.0.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-snapshot": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-30.2.0.tgz", "integrity": "sha512-5WEtTy2jXPFypadKNpbNkZ72puZCa6UjSr/7djeecHWOu7iYhSXSnHScT8wBz3Rn8Ena5d5RYRcsyKIeqG1IyA==", "dev": true, "license": "MIT", "dependencies": { "@babel/core": "^7.27.4", "@babel/generator": "^7.27.5", "@babel/plugin-syntax-jsx": "^7.27.1", "@babel/plugin-syntax-typescript": "^7.27.1", "@babel/types": "^7.27.3", "@jest/expect-utils": "30.2.0", "@jest/get-type": "30.1.0", "@jest/snapshot-utils": "30.2.0", "@jest/transform": "30.2.0", "@jest/types": "30.2.0", "babel-preset-current-node-syntax": "^1.2.0", "chalk": "^4.1.2", "expect": "30.2.0", "graceful-fs": "^4.2.11", "jest-diff": "30.2.0", "jest-matcher-utils": "30.2.0", "jest-message-util": "30.2.0", "jest-util": "30.2.0", "pretty-format": "30.2.0", "semver": "^7.7.2", "synckit": "^0.11.8" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-util": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.2.0.tgz", "integrity": "sha512-QKNsM0o3Xe6ISQU869e+DhG+4CK/48aHYdJZGlFQVTjnbvgpcKyxpzk29fGiO7i/J8VENZ+d2iGnSsvmuHywlA==", "dev": true, "license": "MIT", "dependencies": { "@jest/types": "30.2.0", "@types/node": "*", "chalk": "^4.1.2", "ci-info": "^4.2.0", "graceful-fs": "^4.2.11", "picomatch": "^4.0.2" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-validate": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-30.2.0.tgz", "integrity": "sha512-FBGWi7dP2hpdi8nBoWxSsLvBFewKAg0+uSQwBaof4Y4DPgBabXgpSYC5/lR7VmnIlSpASmCi/ntRWPbv7089Pw==", "dev": true, "license": "MIT", "dependencies": { "@jest/get-type": "30.1.0", "@jest/types": "30.2.0", "camelcase": "^6.3.0", "chalk": "^4.1.2", "leven": "^3.1.0", "pretty-format": "30.2.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-validate/node_modules/camelcase": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", "dev": true, "license": "MIT", "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/jest-watcher": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-30.2.0.tgz", "integrity": "sha512-PYxa28dxJ9g777pGm/7PrbnMeA0Jr7osHP9bS7eJy9DuAjMgdGtxgf0uKMyoIsTWAkIbUW5hSDdJ3urmgXBqxg==", "dev": true, "license": "MIT", "dependencies": { "@jest/test-result": "30.2.0", "@jest/types": "30.2.0", "@types/node": "*", "ansi-escapes": "^4.3.2", "chalk": "^4.1.2", "emittery": "^0.13.1", "jest-util": "30.2.0", "string-length": "^4.0.2" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-worker": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-30.2.0.tgz", "integrity": "sha512-0Q4Uk8WF7BUwqXHuAjc23vmopWJw5WH7w2tqBoUOZpOjW/ZnR44GXXd1r82RvnmI2GZge3ivrYXk/BE2+VtW2g==", "dev": true, "license": "MIT", "dependencies": { "@types/node": "*", "@ungap/structured-clone": "^1.3.0", "jest-util": "30.2.0", "merge-stream": "^2.0.0", "supports-color": "^8.1.1" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-worker/node_modules/supports-color": { "version": "8.1.1", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", "dev": true, "license": "MIT", "dependencies": { "has-flag": "^4.0.0" }, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/chalk/supports-color?sponsor=1" } }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", "dev": true, "license": "MIT" }, "node_modules/js-yaml": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", "dev": true, "license": "MIT", "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "node_modules/jsesc": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", "dev": true, "license": "MIT", "bin": { "jsesc": "bin/jsesc" }, "engines": { "node": ">=6" } }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", "dev": true, "license": "MIT" }, "node_modules/json-parse-even-better-errors": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", "dev": true, "license": "MIT" }, "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", "dev": true, "license": "MIT" }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", "dev": true, "license": "MIT" }, "node_modules/json5": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", "dev": true, "license": "MIT", "bin": { "json5": "lib/cli.js" }, "engines": { "node": ">=6" } }, "node_modules/jsonfile": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.0.tgz", "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==", "dev": true, "license": "MIT", "dependencies": { "universalify": "^2.0.0" }, "optionalDependencies": { "graceful-fs": "^4.1.6" } }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", "dev": true, "license": "MIT", "dependencies": { "json-buffer": "3.0.1" } }, "node_modules/leven": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", "dev": true, "license": "MIT", "engines": { "node": ">=6" } }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", "dev": true, "license": "MIT", "dependencies": { "prelude-ls": "^1.2.1", "type-check": "~0.4.0" }, "engines": { "node": ">= 0.8.0" } }, "node_modules/lines-and-columns": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", "dev": true, "license": "MIT" }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", "dev": true, "license": "MIT", "dependencies": { "p-locate": "^5.0.0" }, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==", "dev": true, "license": "MIT" }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true, "license": "MIT" }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", "dev": true, "license": "ISC", "dependencies": { "yallist": "^3.0.2" } }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", "dev": true, "license": "MIT", "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "node_modules/make-dir": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", "dev": true, "license": "MIT", "dependencies": { "semver": "^7.5.3" }, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", "dev": true, "license": "ISC" }, "node_modules/makeerror": { "version": "1.0.12", "resolved": "https://registry.npmjs.org/makeerror/-/makeerror-1.0.12.tgz", "integrity": "sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==", "dev": true, "license": "BSD-3-Clause", "dependencies": { "tmpl": "1.0.5" } }, "node_modules/merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", "dev": true, "license": "MIT" }, "node_modules/micromatch": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", "dev": true, "license": "MIT", "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" }, "engines": { "node": ">=8.6" } }, "node_modules/micromatch/node_modules/picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", "dev": true, "license": "MIT", "engines": { "node": ">=8.6" }, "funding": { "url": "https://github.com/sponsors/jonschlinkert" } }, "node_modules/mimic-fn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", "dev": true, "license": "MIT", "engines": { "node": ">=6" } }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" }, "engines": { "node": "*" } }, "node_modules/minimist": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" } }, "node_modules/minipass": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", "dev": true, "license": "ISC", "engines": { "node": ">=16 || 14 >=14.17" } }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true, "license": "MIT" }, "node_modules/napi-postinstall": { "version": "0.3.4", "resolved": "https://registry.npmjs.org/napi-postinstall/-/napi-postinstall-0.3.4.tgz", "integrity": "sha512-PHI5f1O0EP5xJ9gQmFGMS6IZcrVvTjpXjz7Na41gTE7eE2hK11lg04CECCYEEjdc17EV4DO+fkGEtt7TpTaTiQ==", "dev": true, "license": "MIT", "bin": { "napi-postinstall": "lib/cli.js" }, "engines": { "node": "^12.20.0 || ^14.18.0 || >=16.0.0" }, "funding": { "url": "https://opencollective.com/napi-postinstall" } }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true, "license": "MIT" }, "node_modules/neo-async": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", "dev": true, "license": "MIT" }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", "dev": true, "license": "MIT" }, "node_modules/node-releases": { "version": "2.0.27", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", "integrity": "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==", "dev": true, "license": "MIT" }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" } }, "node_modules/npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", "dev": true, "license": "MIT", "dependencies": { "path-key": "^3.0.0" }, "engines": { "node": ">=8" } }, "node_modules/onetime": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", "dev": true, "license": "MIT", "dependencies": { "mimic-fn": "^2.1.0" }, "engines": { "node": ">=6" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", "dev": true, "license": "MIT", "dependencies": { "deep-is": "^0.1.3", "fast-levenshtein": "^2.0.6", "levn": "^0.4.1", "prelude-ls": "^1.2.1", "type-check": "^0.4.0", "word-wrap": "^1.2.5" }, "engines": { "node": ">= 0.8.0" } }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", "dev": true, "license": "MIT", "dependencies": { "yocto-queue": "^0.1.0" }, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/p-locate": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", "dev": true, "license": "MIT", "dependencies": { "p-limit": "^3.0.2" }, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", "dev": true, "license": "MIT", "engines": { "node": ">=6" } }, "node_modules/package-json-from-dist": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", "dev": true, "license": "BlueOak-1.0.0" }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", "dev": true, "license": "MIT", "dependencies": { "callsites": "^3.0.0" }, "engines": { "node": ">=6" } }, "node_modules/parse-json": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", "dev": true, "license": "MIT", "dependencies": { "@babel/code-frame": "^7.0.0", "error-ex": "^1.3.1", "json-parse-even-better-errors": "^2.3.0", "lines-and-columns": "^1.1.6" }, "engines": { "node": ">=8" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", "dev": true, "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", "dev": true, "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/path-parse": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true, "license": "MIT" }, "node_modules/path-scurry": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "lru-cache": "^10.2.0", "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" }, "engines": { "node": ">=16 || 14 >=14.18" }, "funding": { "url": "https://github.com/sponsors/isaacs" } }, "node_modules/path-scurry/node_modules/lru-cache": { "version": "10.4.3", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", "dev": true, "license": "ISC" }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", "dev": true, "license": "ISC" }, "node_modules/picomatch": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", "engines": { "node": ">=12" }, "funding": { "url": "https://github.com/sponsors/jonschlinkert" } }, "node_modules/pirates": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.7.tgz", "integrity": "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==", "dev": true, "license": "MIT", "engines": { "node": ">= 6" } }, "node_modules/pkg-dir": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", "dev": true, "license": "MIT", "dependencies": { "find-up": "^4.0.0" }, "engines": { "node": ">=8" } }, "node_modules/pkg-dir/node_modules/find-up": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", "dev": true, "license": "MIT", "dependencies": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" }, "engines": { "node": ">=8" } }, "node_modules/pkg-dir/node_modules/locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", "dev": true, "license": "MIT", "dependencies": { "p-locate": "^4.1.0" }, "engines": { "node": ">=8" } }, "node_modules/pkg-dir/node_modules/p-limit": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", "dev": true, "license": "MIT", "dependencies": { "p-try": "^2.0.0" }, "engines": { "node": ">=6" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/pkg-dir/node_modules/p-locate": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", "dev": true, "license": "MIT", "dependencies": { "p-limit": "^2.2.0" }, "engines": { "node": ">=8" } }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", "dev": true, "license": "MIT", "engines": { "node": ">= 0.8.0" } }, "node_modules/pretty-format": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.2.0.tgz", "integrity": "sha512-9uBdv/B4EefsuAL+pWqueZyZS2Ba+LxfFeQ9DN14HU4bN8bhaxKdkpjpB6fs9+pSjIBu+FXQHImEg8j/Lw0+vA==", "dev": true, "license": "MIT", "dependencies": { "@jest/schemas": "30.0.5", "ansi-styles": "^5.2.0", "react-is": "^18.3.1" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/pretty-format/node_modules/ansi-styles": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", "dev": true, "license": "MIT", "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", "dev": true, "license": "MIT", "engines": { "node": ">=6" } }, "node_modules/pure-rand": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-7.0.1.tgz", "integrity": "sha512-oTUZM/NAZS8p7ANR3SHh30kXB+zK2r2BPcEn/awJIbOvq82WoMN4p62AWWp3Hhw50G0xMsw1mhIBLqHw64EcNQ==", "dev": true, "funding": [ { "type": "individual", "url": "https://github.com/sponsors/dubzzz" }, { "type": "opencollective", "url": "https://opencollective.com/fast-check" } ], "license": "MIT" }, "node_modules/react-is": { "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", "dev": true, "license": "MIT" }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" } }, "node_modules/resolve": { "version": "1.22.11", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz", "integrity": "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ==", "dev": true, "license": "MIT", "dependencies": { "is-core-module": "^2.16.1", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" }, "bin": { "resolve": "bin/resolve" }, "engines": { "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" } }, "node_modules/resolve-cwd": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", "dev": true, "license": "MIT", "dependencies": { "resolve-from": "^5.0.0" }, "engines": { "node": ">=8" } }, "node_modules/resolve-cwd/node_modules/resolve-from": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", "dev": true, "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", "dev": true, "license": "MIT", "engines": { "node": ">=4" } }, "node_modules/rollup": { "version": "4.53.3", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.53.3.tgz", "integrity": "sha512-w8GmOxZfBmKknvdXU1sdM9NHcoQejwF/4mNgj2JuEEdRaHwwF12K7e9eXn1nLZ07ad+du76mkVsyeb2rKGllsA==", "dev": true, "license": "MIT", "peer": true, "dependencies": { "@types/estree": "1.0.8" }, "bin": { "rollup": "dist/bin/rollup" }, "engines": { "node": ">=18.0.0", "npm": ">=8.0.0" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.53.3", "@rollup/rollup-android-arm64": "4.53.3", "@rollup/rollup-darwin-arm64": "4.53.3", "@rollup/rollup-darwin-x64": "4.53.3", "@rollup/rollup-freebsd-arm64": "4.53.3", "@rollup/rollup-freebsd-x64": "4.53.3", "@rollup/rollup-linux-arm-gnueabihf": "4.53.3", "@rollup/rollup-linux-arm-musleabihf": "4.53.3", "@rollup/rollup-linux-arm64-gnu": "4.53.3", "@rollup/rollup-linux-arm64-musl": "4.53.3", "@rollup/rollup-linux-loong64-gnu": "4.53.3", "@rollup/rollup-linux-ppc64-gnu": "4.53.3", "@rollup/rollup-linux-riscv64-gnu": "4.53.3", "@rollup/rollup-linux-riscv64-musl": "4.53.3", "@rollup/rollup-linux-s390x-gnu": "4.53.3", "@rollup/rollup-linux-x64-gnu": "4.53.3", "@rollup/rollup-linux-x64-musl": "4.53.3", "@rollup/rollup-openharmony-arm64": "4.53.3", "@rollup/rollup-win32-arm64-msvc": "4.53.3", "@rollup/rollup-win32-ia32-msvc": "4.53.3", "@rollup/rollup-win32-x64-gnu": "4.53.3", "@rollup/rollup-win32-x64-msvc": "4.53.3", "fsevents": "~2.3.2" } }, "node_modules/rollup-plugin-typescript2": { "version": "0.36.0", "resolved": "https://registry.npmjs.org/rollup-plugin-typescript2/-/rollup-plugin-typescript2-0.36.0.tgz", "integrity": "sha512-NB2CSQDxSe9+Oe2ahZbf+B4bh7pHwjV5L+RSYpCu7Q5ROuN94F9b6ioWwKfz3ueL3KTtmX4o2MUH2cgHDIEUsw==", "dev": true, "license": "MIT", "dependencies": { "@rollup/pluginutils": "^4.1.2", "find-cache-dir": "^3.3.2", "fs-extra": "^10.0.0", "semver": "^7.5.4", "tslib": "^2.6.2" }, "peerDependencies": { "rollup": ">=1.26.3", "typescript": ">=2.4.0" } }, "node_modules/rollup-plugin-typescript2/node_modules/@rollup/pluginutils": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-4.2.1.tgz", "integrity": "sha512-iKnFXr7NkdZAIHiIWE+BX5ULi/ucVFYWD6TbAV+rZctiRTY2PL6tsIKhoIOaoskiWAkgu+VsbXgUVDNLHf+InQ==", "dev": true, "license": "MIT", "dependencies": { "estree-walker": "^2.0.1", "picomatch": "^2.2.2" }, "engines": { "node": ">= 8.0.0" } }, "node_modules/rollup-plugin-typescript2/node_modules/picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", "dev": true, "license": "MIT", "engines": { "node": ">=8.6" }, "funding": { "url": "https://github.com/sponsors/jonschlinkert" } }, "node_modules/semver": { "version": "7.7.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", "dev": true, "license": "ISC", "bin": { "semver": "bin/semver.js" }, "engines": { "node": ">=10" } }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", "dev": true, "license": "MIT", "dependencies": { "shebang-regex": "^3.0.0" }, "engines": { "node": ">=8" } }, "node_modules/shebang-regex": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true, "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/signal-exit": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", "dev": true, "license": "ISC", "engines": { "node": ">=14" }, "funding": { "url": "https://github.com/sponsors/isaacs" } }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", "dev": true, "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", "dev": true, "license": "BSD-3-Clause", "engines": { "node": ">=0.10.0" } }, "node_modules/source-map-support": { "version": "0.5.13", "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", "integrity": "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==", "dev": true, "license": "MIT", "dependencies": { "buffer-from": "^1.0.0", "source-map": "^0.6.0" } }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "dev": true, "license": "BSD-3-Clause" }, "node_modules/stack-utils": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", "integrity": "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==", "dev": true, "license": "MIT", "dependencies": { "escape-string-regexp": "^2.0.0" }, "engines": { "node": ">=10" } }, "node_modules/stack-utils/node_modules/escape-string-regexp": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", "dev": true, "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/string-length": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz", "integrity": "sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==", "dev": true, "license": "MIT", "dependencies": { "char-regex": "^1.0.2", "strip-ansi": "^6.0.0" }, "engines": { "node": ">=10" } }, "node_modules/string-length/node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true, "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/string-length/node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" }, "engines": { "node": ">=8" } }, "node_modules/string-width": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", "dev": true, "license": "MIT", "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" }, "engines": { "node": ">=12" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/string-width-cjs": { "name": "string-width", "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "dev": true, "license": "MIT", "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" }, "engines": { "node": ">=8" } }, "node_modules/string-width-cjs/node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true, "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/string-width-cjs/node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "dev": true, "license": "MIT" }, "node_modules/string-width-cjs/node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" }, "engines": { "node": ">=8" } }, "node_modules/strip-ansi": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.2.tgz", "integrity": "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==", "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^6.0.1" }, "engines": { "node": ">=12" }, "funding": { "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, "node_modules/strip-ansi-cjs": { "name": "strip-ansi", "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" }, "engines": { "node": ">=8" } }, "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true, "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/strip-bom": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", "dev": true, "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/strip-final-newline": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", "dev": true, "license": "MIT", "engines": { "node": ">=6" } }, "node_modules/strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", "dev": true, "license": "MIT", "engines": { "node": ">=8" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", "dev": true, "license": "MIT", "dependencies": { "has-flag": "^4.0.0" }, "engines": { "node": ">=8" } }, "node_modules/supports-preserve-symlinks-flag": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" } }, "node_modules/synckit": { "version": "0.11.11", "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.11.11.tgz", "integrity": "sha512-MeQTA1r0litLUf0Rp/iisCaL8761lKAZHaimlbGK4j0HysC4PLfqygQj9srcs0m2RdtDYnF8UuYyKpbjHYp7Jw==", "dev": true, "license": "MIT", "dependencies": { "@pkgr/core": "^0.2.9" }, "engines": { "node": "^14.18.0 || >=16.0.0" }, "funding": { "url": "https://opencollective.com/synckit" } }, "node_modules/test-exclude": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-7.0.1.tgz", "integrity": "sha512-pFYqmTw68LXVjeWJMST4+borgQP2AyMNbg1BpZh9LbyhUeNkeaPF9gzfPGUAnSMV3qPYdWUwDIjjCLiSDOl7vg==", "dev": true, "license": "ISC", "dependencies": { "@istanbuljs/schema": "^0.1.2", "glob": "^10.4.1", "minimatch": "^9.0.4" }, "engines": { "node": ">=18" } }, "node_modules/test-exclude/node_modules/brace-expansion": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" } }, "node_modules/test-exclude/node_modules/minimatch": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^2.0.1" }, "engines": { "node": ">=16 || 14 >=14.17" }, "funding": { "url": "https://github.com/sponsors/isaacs" } }, "node_modules/tinyglobby": { "version": "0.2.15", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", "dev": true, "license": "MIT", "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.3" }, "engines": { "node": ">=12.0.0" }, "funding": { "url": "https://github.com/sponsors/SuperchupuDev" } }, "node_modules/tmpl": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", "dev": true, "license": "BSD-3-Clause" }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", "dev": true, "license": "MIT", "dependencies": { "is-number": "^7.0.0" }, "engines": { "node": ">=8.0" } }, "node_modules/ts-api-utils": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.1.0.tgz", "integrity": "sha512-CUgTZL1irw8u29bzrOD/nH85jqyc74D6SshFgujOIA7osm2Rz7dYH77agkx7H4FBNxDq7Cjf+IjaX/8zwFW+ZQ==", "dev": true, "license": "MIT", "engines": { "node": ">=18.12" }, "peerDependencies": { "typescript": ">=4.8.4" } }, "node_modules/ts-jest": { "version": "29.4.5", "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.4.5.tgz", "integrity": "sha512-HO3GyiWn2qvTQA4kTgjDcXiMwYQt68a1Y8+JuLRVpdIzm+UOLSHgl/XqR4c6nzJkq5rOkjc02O2I7P7l/Yof0Q==", "dev": true, "license": "MIT", "dependencies": { "bs-logger": "^0.2.6", "fast-json-stable-stringify": "^2.1.0", "handlebars": "^4.7.8", "json5": "^2.2.3", "lodash.memoize": "^4.1.2", "make-error": "^1.3.6", "semver": "^7.7.3", "type-fest": "^4.41.0", "yargs-parser": "^21.1.1" }, "bin": { "ts-jest": "cli.js" }, "engines": { "node": "^14.15.0 || ^16.10.0 || ^18.0.0 || >=20.0.0" }, "peerDependencies": { "@babel/core": ">=7.0.0-beta.0 <8", "@jest/transform": "^29.0.0 || ^30.0.0", "@jest/types": "^29.0.0 || ^30.0.0", "babel-jest": "^29.0.0 || ^30.0.0", "jest": "^29.0.0 || ^30.0.0", "jest-util": "^29.0.0 || ^30.0.0", "typescript": ">=4.3 <6" }, "peerDependenciesMeta": { "@babel/core": { "optional": true }, "@jest/transform": { "optional": true }, "@jest/types": { "optional": true }, "babel-jest": { "optional": true }, "esbuild": { "optional": true }, "jest-util": { "optional": true } } }, "node_modules/ts-jest/node_modules/type-fest": { "version": "4.41.0", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", "dev": true, "license": "(MIT OR CC0-1.0)", "engines": { "node": ">=16" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "dev": true, "license": "0BSD" }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", "dev": true, "license": "MIT", "dependencies": { "prelude-ls": "^1.2.1" }, "engines": { "node": ">= 0.8.0" } }, "node_modules/type-detect": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", "dev": true, "license": "MIT", "engines": { "node": ">=4" } }, "node_modules/type-fest": { "version": "0.21.3", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", "dev": true, "license": "(MIT OR CC0-1.0)", "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" }, "engines": { "node": ">=14.17" } }, "node_modules/typescript-eslint": { "version": "8.48.0", "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.48.0.tgz", "integrity": "sha512-fcKOvQD9GUn3Xw63EgiDqhvWJ5jsyZUaekl3KVpGsDJnN46WJTe3jWxtQP9lMZm1LJNkFLlTaWAxK2vUQR+cqw==", "dev": true, "license": "MIT", "dependencies": { "@typescript-eslint/eslint-plugin": "8.48.0", "@typescript-eslint/parser": "8.48.0", "@typescript-eslint/typescript-estree": "8.48.0", "@typescript-eslint/utils": "8.48.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0", "typescript": ">=4.8.4 <6.0.0" } }, "node_modules/uglify-js": { "version": "3.19.3", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.3.tgz", "integrity": "sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ==", "dev": true, "license": "BSD-2-Clause", "optional": true, "bin": { "uglifyjs": "bin/uglifyjs" }, "engines": { "node": ">=0.8.0" } }, "node_modules/undici-types": { "version": "7.16.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", "dev": true, "license": "MIT" }, "node_modules/universalify": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", "dev": true, "license": "MIT", "engines": { "node": ">= 10.0.0" } }, "node_modules/unrs-resolver": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/unrs-resolver/-/unrs-resolver-1.11.1.tgz", "integrity": "sha512-bSjt9pjaEBnNiGgc9rUiHGKv5l4/TGzDmYw3RhnkJGtLhbnnA/5qJj7x3dNDCRx/PJxu774LlH8lCOlB4hEfKg==", "dev": true, "hasInstallScript": true, "license": "MIT", "dependencies": { "napi-postinstall": "^0.3.0" }, "funding": { "url": "https://opencollective.com/unrs-resolver" }, "optionalDependencies": { "@unrs/resolver-binding-android-arm-eabi": "1.11.1", "@unrs/resolver-binding-android-arm64": "1.11.1", "@unrs/resolver-binding-darwin-arm64": "1.11.1", "@unrs/resolver-binding-darwin-x64": "1.11.1", "@unrs/resolver-binding-freebsd-x64": "1.11.1", "@unrs/resolver-binding-linux-arm-gnueabihf": "1.11.1", "@unrs/resolver-binding-linux-arm-musleabihf": "1.11.1", "@unrs/resolver-binding-linux-arm64-gnu": "1.11.1", "@unrs/resolver-binding-linux-arm64-musl": "1.11.1", "@unrs/resolver-binding-linux-ppc64-gnu": "1.11.1", "@unrs/resolver-binding-linux-riscv64-gnu": "1.11.1", "@unrs/resolver-binding-linux-riscv64-musl": "1.11.1", "@unrs/resolver-binding-linux-s390x-gnu": "1.11.1", "@unrs/resolver-binding-linux-x64-gnu": "1.11.1", "@unrs/resolver-binding-linux-x64-musl": "1.11.1", "@unrs/resolver-binding-wasm32-wasi": "1.11.1", "@unrs/resolver-binding-win32-arm64-msvc": "1.11.1", "@unrs/resolver-binding-win32-ia32-msvc": "1.11.1", "@unrs/resolver-binding-win32-x64-msvc": "1.11.1" } }, "node_modules/update-browserslist-db": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.4.tgz", "integrity": "sha512-q0SPT4xyU84saUX+tomz1WLkxUbuaJnR1xWt17M7fJtEJigJeWUNGUqrauFXsHnqev9y9JTRGwk13tFBuKby4A==", "dev": true, "funding": [ { "type": "opencollective", "url": "https://opencollective.com/browserslist" }, { "type": "tidelift", "url": "https://tidelift.com/funding/github/npm/browserslist" }, { "type": "github", "url": "https://github.com/sponsors/ai" } ], "license": "MIT", "dependencies": { "escalade": "^3.2.0", "picocolors": "^1.1.1" }, "bin": { "update-browserslist-db": "cli.js" }, "peerDependencies": { "browserslist": ">= 4.21.0" } }, "node_modules/uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", "dev": true, "license": "BSD-2-Clause", "dependencies": { "punycode": "^2.1.0" } }, "node_modules/v8-to-istanbul": { "version": "9.3.0", "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", "integrity": "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==", "dev": true, "license": "ISC", "dependencies": { "@jridgewell/trace-mapping": "^0.3.12", "@types/istanbul-lib-coverage": "^2.0.1", "convert-source-map": "^2.0.0" }, "engines": { "node": ">=10.12.0" } }, "node_modules/walker": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", "integrity": "sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==", "dev": true, "license": "Apache-2.0", "dependencies": { "makeerror": "1.0.12" } }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", "dev": true, "license": "ISC", "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "bin/node-which" }, "engines": { "node": ">= 8" } }, "node_modules/word-wrap": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" } }, "node_modules/wordwrap": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", "dev": true, "license": "MIT" }, "node_modules/wrap-ansi": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^6.1.0", "string-width": "^5.0.1", "strip-ansi": "^7.0.1" }, "engines": { "node": ">=12" }, "funding": { "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, "node_modules/wrap-ansi-cjs": { "name": "wrap-ansi", "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" }, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true, "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "dev": true, "license": "MIT" }, "node_modules/wrap-ansi-cjs/node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "dev": true, "license": "MIT", "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" }, "engines": { "node": ">=8" } }, "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" }, "engines": { "node": ">=8" } }, "node_modules/wrap-ansi/node_modules/ansi-styles": { "version": "6.2.3", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", "dev": true, "license": "MIT", "engines": { "node": ">=12" }, "funding": { "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, "node_modules/write-file-atomic": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-5.0.1.tgz", "integrity": "sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==", "dev": true, "license": "ISC", "dependencies": { "imurmurhash": "^0.1.4", "signal-exit": "^4.0.1" }, "engines": { "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", "dev": true, "license": "ISC", "engines": { "node": ">=10" } }, "node_modules/yallist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", "dev": true, "license": "ISC" }, "node_modules/yargs": { "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", "dev": true, "license": "MIT", "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" }, "engines": { "node": ">=12" } }, "node_modules/yargs-parser": { "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", "dev": true, "license": "ISC", "engines": { "node": ">=12" } }, "node_modules/yargs/node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true, "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/yargs/node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "dev": true, "license": "MIT" }, "node_modules/yargs/node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "dev": true, "license": "MIT", "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" }, "engines": { "node": ">=8" } }, "node_modules/yargs/node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" }, "engines": { "node": ">=8" } }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", "dev": true, "license": "MIT", "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } } } } xgrammar-0.2.3/web/package.json000066400000000000000000000023621521764210300164010ustar00rootroot00000000000000{ "name": "@mlc-ai/web-xgrammar", "version": "0.1.27", "description": "", "main": "lib/index.js", "types": "lib/index.d.ts", "type": "module", "scripts": { "build": "./build.sh; rollup -c", "lint": "npx eslint ./src/ ./tests/ ./example/", "test": "./run_test.sh" }, "files": [ "lib" ], "repository": { "type": "git", "url": "git+https://github.com/mlc-ai/xgrammar" }, "keywords": [ "machine_learning", "llm", "nlp" ], "license": "Apache-2.0", "homepage": "https://github.com/mlc-ai/xgrammar/tree/main/web", "devDependencies": { "@eslint/js": "^9.9.0", "@eslint/eslintrc": "^3.3.1", "@jest/globals": "^30.2.0", "@rollup/plugin-commonjs": "^29.0.0", "@rollup/plugin-node-resolve": "^16.0.3", "@rollup/plugin-wasm": "^6.2.2", "@types/jest": "^30.0.0", "@typescript-eslint/parser": "8.48.0", "@typescript-eslint/eslint-plugin": "8.48.0", "eslint": "^9.39.1", "jest": "^30.2.0", "rollup": "^4.53.3", "rollup-plugin-typescript2": "^0.36.0", "ts-jest": "^29.4.5", "tslib": "^2.3.1", "typescript-eslint": "^8.47.0", "typescript": "^5.9.3", "@mlc-ai/web-tokenizers": "^0.1.6" }, "overrides": { "test-exclude": "^7.0.1" } } xgrammar-0.2.3/web/rollup.config.js000066400000000000000000000011771521764210300172350ustar00rootroot00000000000000import { nodeResolve } from '@rollup/plugin-node-resolve'; import commonjs from '@rollup/plugin-commonjs'; import { wasm } from '@rollup/plugin-wasm'; import typescript from 'rollup-plugin-typescript2' export default { input: 'src/index.ts', output: [ { file: 'lib/index.js', exports: 'named', name: 'xgrammar', format: 'umd', sourcemap: true } ], plugins: [ nodeResolve({ browser: true }), commonjs(), wasm(), typescript({ rollupCommonJSResolveHack: false, clean: true }) ] }; xgrammar-0.2.3/web/run_test.sh000077500000000000000000000013661521764210300163200ustar00rootroot00000000000000#!/bin/bash set -euxo pipefail # Ensure latest TypeScript output exists before running Jest ESM suite. npx tsc -p tsconfig.json # The wasm binding is produced by emcc in src/. Copy this into lib/ so # package entry (which points at lib/) can load it during tests. if [[ ! -f src/xgrammar_binding.js ]]; then echo "Missing src/xgrammar_binding.js. Run ./build.sh first." >&2 exit 1 fi cp -f src/xgrammar_binding.js lib/xgrammar_binding.js # tokenizers ships CommonJS-in-ESM wrapping. Copy library file to .cjs # so Node will load it through CommonJS loader during tests. tokenizers_lib="node_modules/@mlc-ai/web-tokenizers/lib" cp -f "${tokenizers_lib}/index.js" "${tokenizers_lib}/index.cjs" node --experimental-vm-modules node_modules/jest/bin/jest xgrammar-0.2.3/web/src/000077500000000000000000000000001521764210300146775ustar00rootroot00000000000000xgrammar-0.2.3/web/src/index.ts000066400000000000000000000012451521764210300163600ustar00rootroot00000000000000import { Grammar, GrammarCompiler, CompiledGrammar, GrammarMatcher, TokenizerInfo, Testings, StructuralTagItem } from "./xgrammar.js"; export { Grammar, GrammarCompiler, CompiledGrammar, GrammarMatcher, TokenizerInfo, Testings, StructuralTagItem } export type { AnyTextFormat, ConstStringFormat, GrammarFormat, JSONSchemaFormat, OrFormat, QwenXMLParameterFormat, RegexFormat, SequenceFormat, StructuralTag, StructuralTagFormat, StructuralTagLike, TagFormat, TagsWithSeparatorFormat, TriggeredTagsFormat, } from "./xgrammar.js" export default { Grammar, GrammarCompiler, CompiledGrammar, GrammarMatcher, TokenizerInfo, Testings, StructuralTagItem } xgrammar-0.2.3/web/src/structural_tag.ts000066400000000000000000000150611521764210300203150ustar00rootroot00000000000000/** * Type definitions and helpers for structural tag support in web bindings. */ export type JSONSchemaValue = boolean | Record; export interface ConstStringFormat { type: "const_string"; value: string; } export interface JSONSchemaFormat { type: "json_schema"; json_schema: JSONSchemaValue; } export interface QwenXMLParameterFormat { type: "qwen_xml_parameter"; json_schema: JSONSchemaValue; } export interface AnyTextFormat { type: "any_text"; } export interface GrammarFormat { type: "grammar"; grammar: string; } export interface RegexFormat { type: "regex"; pattern: string; } export interface SequenceFormat { type: "sequence"; elements: StructuralTagFormat[]; } export interface OrFormat { type: "or"; elements: StructuralTagFormat[]; } export interface TagFormat { type: "tag"; begin: string; content: StructuralTagFormat; end: string; } export interface TriggeredTagsFormat { type: "triggered_tags"; triggers: string[]; tags: TagFormat[]; at_least_one?: boolean; stop_after_first?: boolean; } export interface TagsWithSeparatorFormat { type: "tags_with_separator"; tags: TagFormat[]; separator: string; at_least_one?: boolean; stop_after_first?: boolean; } export type StructuralTagFormat = | AnyTextFormat | ConstStringFormat | JSONSchemaFormat | GrammarFormat | RegexFormat | QwenXMLParameterFormat | OrFormat | SequenceFormat | TagFormat | TriggeredTagsFormat | TagsWithSeparatorFormat; export interface StructuralTag { /** * The discriminant for structural tags. Optional to keep compatibility with the Python API. */ type?: "structural_tag"; /** * The structural tag format definition. */ format: StructuralTagFormat; } export type StructuralTagLike = StructuralTag | StructuralTagFormat | Record | string; /** * A structural tag item used in the deprecated legacy API. */ export class StructuralTagItem { constructor( public begin: string, public schema: string | JSONSchemaValue, public end: string, ) { } /** * Convert the schema to a string representation. * * @returns The schema as a JSON string. */ getSchemaAsString(): string { if (typeof this.schema === "string") { return this.schema; } return JSON.stringify(this.schema); } /** * Convert the schema to a JSON object or boolean. */ getSchemaAsJSON(): JSONSchemaValue { if (typeof this.schema === "boolean") { return this.schema; } if (typeof this.schema === "string") { try { const parsed = JSON.parse(this.schema); if (typeof parsed === "boolean" || isPlainObject(parsed)) { return parsed as JSONSchemaValue; } throw new Error("Schema string must be a JSON object or boolean."); } catch (err) { throw new Error( `Failed to parse schema string "${this.schema}": ${err instanceof Error ? err.message : String(err)}`, ); } } if (isPlainObject(this.schema)) { return this.schema as Record; } throw new Error("Schema must be either a JSON string, object, or boolean."); } } /** * Convert legacy structural tag items to a StructuralTag definition. */ export function structuralTagFromLegacy(tags: StructuralTagItem[], triggers: string[]): StructuralTag { return { type: "structural_tag", format: { type: "triggered_tags", triggers: [...triggers], tags: tags.map((tag) => ({ type: "tag", begin: tag.begin, end: tag.end, content: { type: "json_schema", json_schema: tag.getSchemaAsJSON(), }, })), }, }; } /** * Normalize user input to a structural tag JSON string. */ export function structuralTagArgsToJSONString( structuralTagOrLegacy: StructuralTagLike | StructuralTagItem[], triggers?: string[], ): string { if (Array.isArray(structuralTagOrLegacy)) { if (!Array.isArray(triggers)) { throw new Error("Legacy structural tag usage requires both tags and triggers."); } const structuralTag = structuralTagFromLegacy(structuralTagOrLegacy, triggers); return JSON.stringify(structuralTag); } if (Array.isArray(triggers)) { throw new Error("The triggers argument is only supported with legacy StructuralTagItem usage."); } return structuralTagToJSONString(structuralTagOrLegacy); } const STRUCTURAL_TAG_FORMAT_TYPES = new Set([ "any_text", "const_string", "grammar", "json_schema", "qwen_xml_parameter", "regex", "sequence", "or", "tag", "triggered_tags", "tags_with_separator", ]); function structuralTagToJSONString(structuralTag: StructuralTagLike): string { if (typeof structuralTag === "string") { return structuralTag; } const normalized = structuralTagToObject(structuralTag); return JSON.stringify({ type: "structural_tag", format: normalized.format, }); } function structuralTagToObject(structuralTag: Exclude): StructuralTag { if (isPlainObject(structuralTag)) { if ("format" in structuralTag) { return normalizeStructuralTagObject(structuralTag); } if (isStructuralTagFormat(structuralTag)) { return { type: "structural_tag", format: structuralTag }; } } if (isStructuralTagFormat(structuralTag)) { return { type: "structural_tag", format: structuralTag }; } throw new Error("Structural tag input must be a JSON string, StructuralTag object, or format."); } function normalizeStructuralTagObject(value: Record): StructuralTag { if (!("format" in value)) { throw new Error('Structural tag object must contain a "format" field.'); } const typeField = value["type"]; if (typeField !== undefined && typeField !== "structural_tag") { throw new Error('Structural tag "type" field must be "structural_tag" when present.'); } const formatCandidate = (value as { format: unknown }).format; if (!isStructuralTagFormat(formatCandidate)) { throw new Error('Structural tag "format" field must be a valid structural tag format.'); } return { type: "structural_tag", format: formatCandidate, }; } function isStructuralTagFormat(value: unknown): value is StructuralTagFormat { if (!isPlainObject(value)) { return false; } const typeValue = (value as { type?: unknown }).type; if (typeof typeValue !== "string") { return false; } return STRUCTURAL_TAG_FORMAT_TYPES.has(typeValue); } function isPlainObject(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } xgrammar-0.2.3/web/src/xgrammar.ts000066400000000000000000000711021521764210300170660ustar00rootroot00000000000000import Module from "./xgrammar_binding.js"; import { StructuralTagItem as StructuralTagItemImpl, structuralTagArgsToJSONString, } from "./structural_tag.js"; import type { StructuralTagItem, StructuralTagLike, } from "./structural_tag.js"; export { StructuralTagItemImpl as StructuralTagItem }; export type { AnyTextFormat, ConstStringFormat, GrammarFormat, JSONSchemaFormat, QwenXMLParameterFormat, RegexFormat, SequenceFormat, StructuralTag, StructuralTagFormat, StructuralTagLike, TagFormat, TagsWithSeparatorFormat, TriggeredTagsFormat, OrFormat, } from "./structural_tag.js"; let binding: any = null; async function asyncInitBinding() { if (binding == null) { binding = await Module(); } } type SeparatorPair = { first: string; second: string }; function toSeparatorPair(separators?: [string, string]): SeparatorPair | undefined { if (separators === undefined) { return undefined; } const [comma, colon] = separators; if (comma === undefined || colon === undefined) { throw new Error("Argument separators must contain exactly two string entries."); } return { first: comma, second: colon }; } /** * Various testing methods that are not optimized for performance. */ export class Testings { /** * Convert JSON schema string to EBNF grammar string. For test purposes. * * @param {string} schema The schema string. * @param {boolean} [anyWhitespace=true] Whether to ignore the indentation * restrictions, and allow any whitespace. * @param {number} [indent=2] The number of spaces for indentation. If -1, the grammar will * enforce the output to be in one line. * @param {[string, string]} [separators] Two separators that will be enforced by the grammar: * comma and colon. Examples: (",", ":"), (", ", ": "). If undefined, the default separators will * be used: (",", ": ") when the indent is not undefined, and (", ", ": ") otherwise. This follows * the convention in Python's json.dumps(). * @param {boolean} [strictMode=true] Whether to use strict mode. In strict mode, the generated * grammar will not allow properties and items that is not specified in the schema. This is * equivalent to setting unevaluatedProperties and unevaluatedItems to false. * @param {number} [maxWhitespaceCnt] Maximum number of whitespace characters allowed between * JSON elements when anyWhitespace is true. Undefined means unlimited. * @param {"json" | "xml"} [jsonFormat="json"] The JSON schema output format. * @param {boolean} [anyOrder=false] Whether to allow object properties to appear in any order. * @returns {string} The EBNF grammar string. */ static async _jsonSchemaToEBNF( schema: string, anyWhitespace: boolean = true, indent = 2, separators?: [string, string], strictMode = true, maxWhitespaceCnt?: number, jsonFormat: "json" | "xml" = "json", anyOrder: boolean = false ): Promise { const separatorsPair = toSeparatorPair(separators); await asyncInitBinding(); // indent being -1 is equivalent to not having a value for the std::optional arg in C++. // This is a workaround to Typescript not being able to express Optional value like Python; if // user specifies indent to be undefined, it still becomes 2. const optionalIndent: number | undefined = indent == -1 ? undefined : indent; const formatEnum = jsonFormat === "xml" ? binding.JSONFormat.kQwenXML : binding.JSONFormat.kJSON; return binding._JSONSchemaToEBNF( schema, anyWhitespace, optionalIndent, separatorsPair, strictMode, maxWhitespaceCnt, formatEnum, anyOrder ); } /** * * @param {Int32Array} bitmask Bitmask returned by getNextTokenBitmask(). * @param {number} vocabSize Vocab size returned by getVocabSize(). * @param {number} index The batch index of the bitmask. For batch inference, bitmask[index] will * be used. Defaults to 0. * @returns An array of vocab ID that will be rejected as a result of the bitmask. */ static async debugGetMaskedTokensFromBitmask( bitmask: Int32Array, vocabSize: number, index: number = 0, ): Promise { await asyncInitBinding(); const bitmaskIntVector = binding.vecIntFromJSArray(bitmask); const rejectedIDsIntVector = binding.DebugGetMaskedTokensFromBitmask( bitmaskIntVector, vocabSize, index ); bitmaskIntVector.delete(); const rejectedIDsInt32Array = binding.vecIntToView(rejectedIDsIntVector).slice(); rejectedIDsIntVector.delete(); return rejectedIDsInt32Array; } /** * Convert a BNF grammar string to a Grammar object without normalization. For test * purposes. The result grammar cannot be compiled / used in GrammarMatcher. * * @param {string} ebnfString The EBNF grammar string to be converted. * @param {string} [rootRule="root"] The name of the root rule. Default: "root". * @returns {Grammar} The unnormalized Grammar object converted from the input BNF grammar string. */ static async ebnfToGrammarNoNormalization( ebnfString: string, rootRule: string = "root" ): Promise { await asyncInitBinding(); return new Grammar(new binding.EBNFToGrammarNoNormalization(ebnfString, rootRule)); } /** * Given a grammar and an input string, check if the grammar accepts the input string. * No tokenizer information is needed. Only for testing purposes. */ static async isGrammarAcceptString( grammar: Grammar, inputString: string, debugPrint: boolean = false, ): Promise { await asyncInitBinding(); const tokenizerInfo = await TokenizerInfo.createTokenizerInfo([]); const grammarCompiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo, false); const compiledGrammar = await grammarCompiler.compileGrammar(grammar); const matcher = await GrammarMatcher.createGrammarMatcher(compiledGrammar, undefined, true); const accepted = matcher._acceptString(inputString, debugPrint); const isTerminated = matcher.isTerminated(); matcher.dispose(); compiledGrammar.dispose(); grammarCompiler.dispose(); tokenizerInfo.dispose(); if (!accepted) { return false; } return isTerminated; } } /** * This class stores the abstract syntax tree (AST) of the Backus-Naur Form (BNF) grammar and * provides utilities to parse and print the AST. User should provide a BNF/EBNF (Extended * Backus-Naur Form) grammar, and use from_ebnf_string to parse and simplify the grammar into an * AST of BNF grammar. */ export class Grammar { handle: any; /** * @internal * Private constructor. Factory methods are used since binding initialization is asynchronous. * @param {any} handle handle of Grammar created by binding. */ constructor(handle: any) { this.handle = handle; } /** * Dispose this Grammar. */ dispose() { this.handle.delete(); } /** * Construct a BNF grammar with a EBNF-formatted string. The grammar will be normalized * (simplified) by default. * EBNF grammar: see https://www.w3.org/TR/xml/#sec-notation. Note: * 1. Use # as the comment mark * 2. Use C-style unicode escape sequence \u01AB, \U000001AB, \xAB * 3. A-B (match A and not match B) is not supported yet * 4. Lookahead assertion can be added at the end of a rule to speed up matching. E.g. * ``` * root ::= "ab" a [a-z] * a ::= "cd" (=[a-z]) * ``` * The assertion (=[a-z]) means a must be followed by [a-z]. * @param {string} ebnfString The grammar string * @param {string} [rootRule="root"] The name of the root rule. Default: "root". * @returns {Grammar} The parsed BNF grammar. */ static async fromEBNF(ebnfString: string, rootRule = "root"): Promise { await asyncInitBinding(); return new Grammar(new binding.Grammar.FromEBNF(ebnfString, rootRule)); } /** * Get the grammar of standard JSON. * @returns {Grammar} The JSON grammar. */ static async builtinJSONGrammar(): Promise { await asyncInitBinding(); return new Grammar(new binding.Grammar.BuiltinJSONGrammar()); } /** * Construct a BNF grammar from the json schema string. The schema string should be in the * format of the schema of a JSON file. We will parse the schema and generate a BNF grammar. * * @param {string} schema The schema string. * @param {boolean} [anyWhitespace=true] Whether to ignore the indentation * restrictions, and allow any whitespace. * @param {number} [indent=2] The number of spaces for indentation. If -1, the grammar will * enforce the output to be in one line. * @param {[string, string]} [separators] Two separators that will be enforced by the grammar: * comma and colon. Examples: (",", ":"), (", ", ": "). If undefined, the default separators will * be used: (",", ": ") when the indent is not undefined, and (", ", ": ") otherwise. This follows * the convention in Python's json.dumps(). * @param {boolean} [strictMode=true] Whether to use strict mode. In strict mode, the generated * grammar will not allow properties and items that is not specified in the schema. This is * equivalent to setting unevaluatedProperties and unevaluatedItems to false. * @param {number} [maxWhitespaceCnt] Maximum number of whitespace characters allowed between * JSON elements when anyWhitespace is true. Undefined means unlimited. * @param {boolean} [anyOrder=false] Whether to allow object properties to appear in any order. * @returns {Grammar} The generated BNF grammar. */ static async fromJSONSchema( schema: string, anyWhitespace: boolean = true, indent = 2, separators?: [string, string], strictMode = true, maxWhitespaceCnt?: number, anyOrder: boolean = false ): Promise { const separatorsPair = toSeparatorPair(separators); await asyncInitBinding(); // indent being -1 is equivalent to not having a value for the std::optional arg in C++. // This is a workaround to Typescript not being able to express Optional value like Python; if // user specifies indent to be undefined, it still becomes 2. const optionalIndent: number | undefined = indent == -1 ? undefined : indent; // print_converted_ebnf is always false for the web version const printConvertedEBNF = false; return new Grammar( new binding.Grammar.FromJSONSchema( schema, anyWhitespace, optionalIndent, separatorsPair, strictMode, maxWhitespaceCnt, printConvertedEBNF, anyOrder )); } /** * Print the BNF grammar to a string, in standard BNF format. * @returns The BNF grammar string. */ toString(): string { return this.handle.ToString(); } /** * Create a grammar from structural tags. The structural tag handles the dispatching * of different grammars based on the tags and triggers: it initially allows any output, * until a trigger is encountered, then dispatch to the corresponding tag; when the end tag * is encountered, the grammar will allow any following output, until the next trigger is * encountered. * The tags parameter is used to specify the output pattern. It is especially useful for LLM * function calling, where the pattern is: * {"arg1": ..., "arg2": ...}. * This pattern consists of three parts: a begin tag (), a parameter list * according to some schema ({"arg1": ..., "arg2": ...}), and an end tag (). This * pattern can be described in a StructuralTagItem with a begin tag, a schema, and an end tag. * The structural tag is able to handle multiple such patterns by passing them into multiple * tags. * * The triggers parameter is used to trigger the dispatching of different grammars. The trigger * should be a prefix of a provided begin tag. When the trigger is encountered, the * corresponding tag should be used to constrain the following output. There can be multiple * tags matching the same trigger. Then if the trigger is encountered, the following output * should match one of the tags. For example, in function calling, the triggers can be * ["). * * The correspondence of tags and triggers is automatically determined: all tags with the * same trigger will be grouped together. User should make sure any trigger is not a prefix * of another trigger: then the correspondence of tags and triggers will be ambiguous. * * To use this grammar in grammar-guided generation, the GrammarMatcher constructed from * structural tag will generate a mask for each token. When the trigger is not encountered, * the mask will likely be all-1 and not have to be used (fill_next_token_bitmask returns * False, meaning no token is masked). When a trigger is encountered, the mask should be * enforced (fill_next_token_bitmask will return True, meaning some token is masked) to the * output logits. * * The benefit of this method is the token boundary between tags and triggers is automatically * handled. The user does not need to worry about the token boundary. * * There are two ways to construct a structural tag grammar: * * 1. Preferred: `Grammar.fromStructuralTag(structuralTag)` where `structuralTag` can be a JSON * string, a `StructuralTag` object, or any of the structural tag format objects. * 2. Deprecated: `Grammar.fromStructuralTag(tags, triggers)` which accepts the legacy * `StructuralTagItem[]` and trigger prefixes. * * @param {StructuralTag | string | StructuralTagFormat | StructuralTagItem[]} structuralTagOrTags * The structural tag definition or legacy structural tag items. * @param {string[]} [triggers] Deprecated trigger list when using the legacy API. * @returns {Grammar} The grammar created from the structural tag definition. */ static async fromStructuralTag(structuralTag: StructuralTagLike): Promise; static async fromStructuralTag(tags: StructuralTagItem[], triggers: string[]): Promise; static async fromStructuralTag( structuralTagOrTags: StructuralTagLike | StructuralTagItem[], triggers?: string[], ): Promise { await asyncInitBinding(); const structuralTagJSON = structuralTagArgsToJSONString(structuralTagOrTags, triggers); return new Grammar(binding.Grammar.FromStructuralTag(structuralTagJSON)); } } /** * A class that wraps a preprocessed vocab, needed to instantiate GrammarCompiler. */ export class TokenizerInfo { handle: any; /** * @internal * Private constructor. Factory methods are used since binding initialization is asynchronous. * @param {any} handle handle of TokenizerInfo created by binding. */ constructor(handle: any) { this.handle = handle; }; /** * Dispose this tokenizer info object. */ dispose() { this.handle.delete(); } /** * Get the vocab size. */ getVocabSize(): number { return this.handle.GetVocabSize(); } /** * Get the post-processed vocab. Returned as a handle of type binding.VectorString */ getDecodedVocabHandle(): any { return this.handle.GetDecodedVocab(); } /** * Instantiate with raw vocab and the vocab type by internally post-processing * the raw vocab by decoding each token with the provided vocab type. * @param {string[]} encodedVocab: the vocab in the form of a string list of tokens, * ordered by their token id. It should include all the special tokens. * @param {string} vocabType: either "byte_fallback", "byte_level", or `raw`. See `tokenizer.cc` * for its semantic. * @param {boolean} prependSpaceInTokenization: whether the tokenizer will prepend a space before * the text in the tokenization process. * @param {number} vocabSize: the full vocab size read from `config.json`. If not provided, will * use length of `encodedVocab`. Note some model has a vocab size larger in `config.json` due * to padding. Essentially the size of the logits. * @param {number[] | number} [stopTokenIds=undefined] Stop tokens to override the default ones. */ static async createTokenizerInfo( encodedVocab: string[], vocabType: string = "raw", prependSpaceInTokenization: boolean = false, vocabSize?: number, stopTokenIds?: number[] | number, ): Promise { await asyncInitBinding(); // Convert string[] to std::vector const encodedVocabVec = binding.vecStringFromJSArray(encodedVocab); // Convert stopTokenIds to std::vector if not undefined if (stopTokenIds !== undefined) { if (!Array.isArray(stopTokenIds)) { stopTokenIds = [stopTokenIds]; } stopTokenIds = binding.vecIntFromJSArray(stopTokenIds); } // Instantiate TokenizerInfo return new TokenizerInfo(new binding.TokenizerInfo( encodedVocabVec, vocabType.toUpperCase(), vocabSize, stopTokenIds, prependSpaceInTokenization, )); } } export class CompiledGrammar { handle: any; /** * @internal * Private constructor. Factory methods are used since binding initialization is asynchronous. * @param {any} handle handle of CompiledGrammar created by binding. */ constructor(handle: any) { this.handle = handle; }; /** * Dispose this compiled grammar object. */ dispose() { this.handle.delete(); } /** * @returns {Grammar} The grammar used to compile this CompiledGrammar. */ grammar(): Grammar { return new Grammar(this.handle.GetGrammar()); } /** * @returns {TokenizerInfo} The tokenizer info used to compile this CompiledGrammar. */ tokenizerInfo(): TokenizerInfo { return new TokenizerInfo(this.handle.GetTokenizerInfo()); } } export class GrammarCompiler { handle: any; /** * @internal * Private constructor. Factory methods are used since binding initialization is asynchronous. * @param {any} handle handle of GrammarCompiler created by binding. */ private constructor(handle: any) { this.handle = handle; }; /** * Dispose this grammar compiler object. */ dispose() { this.handle.delete(); }; /** * * @param tokenizerInfo {TokenizerInfo} The tokenizer info that contains preprocessed vocab. * @param cacheEnabled {boolean} Whether to enable caching. Default is true. */ static async createGrammarCompiler( tokenizerInfo: TokenizerInfo, cacheEnabled: boolean = true, ): Promise { await asyncInitBinding(); // NOTE(Charlie): Have not figured out how to do multithreading in WASM, so always set to 1. return new GrammarCompiler(new binding.GrammarCompiler( tokenizerInfo.handle, /**max_threads=*/1, cacheEnabled )); } /** * Get CompiledGrammar from the json schema string. The schema string should be in the * format of the schema of a JSON file. We will parse the schema and generate a BNF grammar. * * @param {string} schema The schema string. * @param {boolean} [anyWhitespace=true] Whether to ignore the indentation * restrictions, and allow any whitespace. * @param {number} [indent=2] The number of spaces for indentation. If -1, the grammar will * enforce the output to be in one line. * @param {[string, string]} [separators] Two separators that will be enforced by the grammar: * comma and colon. Examples: (",", ":"), (", ", ": "). If undefined, the default separators will * be used: (",", ": ") when the indent is not undefined, and (", ", ": ") otherwise. This follows * the convention in Python's json.dumps(). * @param {boolean} [strictMode=true] Whether to use strict mode. In strict mode, the generated * grammar will not allow properties and items that is not specified in the schema. This is * equivalent to setting unevaluatedProperties and unevaluatedItems to false. * @param {number} [maxWhitespaceCnt] Maximum number of whitespace characters allowed between * JSON elements when anyWhitespace is true. Undefined means unlimited. * @param {boolean} [anyOrder=false] Whether to allow object properties to appear in any order. * @returns {CompiledGrammar} The compiled grammar for the specified JSON schema. */ async compileJSONSchema( schema: string, anyWhitespace: boolean = true, indent = 2, separators?: [string, string], strictMode = true, maxWhitespaceCnt?: number, anyOrder: boolean = false ): Promise { const separatorsPair = toSeparatorPair(separators); await asyncInitBinding(); // indent being -1 is equivalent to not having a value for the std::optional arg in C++. // This is a workaround to Typescript not being able to express Optional value like Python; if // user specifies indent to be undefined, it still becomes 2. const optionalIndent: number | undefined = indent == -1 ? undefined : indent; return new CompiledGrammar( this.handle.CompileJSONSchema( schema, anyWhitespace, optionalIndent, separatorsPair, strictMode, maxWhitespaceCnt, anyOrder)); } /** * @returns {CompiledGrammar} The compiled grammar for JSON. */ async compileBuiltinJSONGrammar(): Promise { await asyncInitBinding(); return new CompiledGrammar(this.handle.CompileBuiltinJSONGrammar()); } /** * Get CompiledGrammar from the EBNF-formatted string. The grammar will be normalized * (simplified) by default. * EBNF grammar: see https://www.w3.org/TR/xml/#sec-notation. Note: * 1. Use # as the comment mark * 2. Use C-style unicode escape sequence \u01AB, \U000001AB, \xAB * 3. A-B (match A and not match B) is not supported yet * 4. Lookahead assertion can be added at the end of a rule to speed up matching. E.g. * ``` * root ::= "ab" a [a-z] * a ::= "cd" (=[a-z]) * ``` * The assertion (=[a-z]) means a must be followed by [a-z]. * @param {string} ebnfString The grammar string * @param {string} [rootRule="root"] The name of the root rule. Default: "root". * @returns {CompiledGrammar} The compiled grammar for the specified EBNF string. */ async compileGrammar(grammar: Grammar): Promise; async compileGrammar(grammar: string, rootRule?: string): Promise; async compileGrammar(grammar: string | Grammar, rootRule: string = "root"): Promise { await asyncInitBinding(); if (typeof grammar === "string") { const grammarObj = await Grammar.fromEBNF(grammar, rootRule); return new CompiledGrammar(this.handle.CompileGrammar(grammarObj.handle)); } else { return new CompiledGrammar(this.handle.CompileGrammar(grammar.handle)); } } /** * Compile a grammar from structural tags. See Grammar.fromStructuralTag() for more details. * * @param {StructuralTag | string | StructuralTagFormat | StructuralTagItem[]} structuralTagOrTags * The structural tag definition or legacy structural tag items. * @param {string[]} [triggers] Deprecated trigger list when using the legacy API. * @returns {CompiledGrammar} The compiled grammar. */ async compileStructuralTag(structuralTag: StructuralTagLike): Promise; async compileStructuralTag(tags: StructuralTagItem[], triggers: string[]): Promise; async compileStructuralTag( structuralTagOrTags: StructuralTagLike | StructuralTagItem[], triggers?: string[], ): Promise { await asyncInitBinding(); const structuralTagJSON = structuralTagArgsToJSONString(structuralTagOrTags, triggers); return new CompiledGrammar(this.handle.CompileStructuralTag(structuralTagJSON)); } } /** * A stateful matcher to match tokens to the specified BNF grammar. This class is the core logic * of the grammar-guided generation. * * This class implements the non-deterministic pushdown automaton (NPDA) matching algorithm to * match characters to a BNF grammar. It keep track of the current state of the matching process by * maintaining several stacks internally as possible paths in the NPDA. It also supports * backtracking. * * It is particularly capable of finding the set of tokens that are acceptable for the next step * and storing them in a bitmask. This aids in grammar-guided generation. */ export class GrammarMatcher { private handle: any; private vocab_size: number; /** * @internal * Private constructor. Factory methods are used since binding initialization is asynchronous. * @param {any} handle handle of GrammarMatcher created by binding. */ private constructor(handle: any, vocab_size: number) { this.handle = handle; this.vocab_size = vocab_size; } /** * Dispose this grammar state matcher. */ dispose() { this.handle.delete(); } /** * Construct a GrammarMatcher. * @param {CompiledGrammar} compiledGrammar A compiled grammar from GrammarCompiler. * @param {number[] | number} [overrideStopTokens=undefined] Stop tokens to override the default ones. * @param {boolean} [terminateWithoutStopToken=false] Whether to terminate without stop token. * @param {number} [maxRollbackTokens=-1] [Deprecated] Max rollback tokens. * @returns {GrammarMatcher} The constructed GrammarMatcher. */ static async createGrammarMatcher( compiledGrammar: CompiledGrammar, overrideStopTokens?: number[] | number, terminateWithoutStopToken: boolean = false, maxRollbackTokens: number = -1, ): Promise { await asyncInitBinding(); // Convert overrideStopTokens to std::vector if not undefined if (overrideStopTokens !== undefined) { if (!Array.isArray(overrideStopTokens)) { overrideStopTokens = [overrideStopTokens]; } overrideStopTokens = binding.vecIntFromJSArray(overrideStopTokens); } return new GrammarMatcher(new binding.GrammarMatcher( compiledGrammar.handle, overrideStopTokens, terminateWithoutStopToken, maxRollbackTokens, ), compiledGrammar.tokenizerInfo().getVocabSize()); } /** * Get the maximum number of rollback tokens allowed. */ getMaxRollbackTokens(): number { return this.handle.GetMaxRollbackTokens(); } /** * Accept one token and update the state of the matcher. * @param {number} tokenID The id of the token to accept. * @param {boolean} [verbose=false] To print debugging info * @returns {boolean} Whether the token is accepted. */ acceptToken(tokenID: number, verbose: boolean = false): boolean { return this.handle.AcceptToken(tokenID, verbose); } /** * Accept one unicode codepoint to the current state. For test purposes. * @param {string} inputStr The unicode codepoint of the character to be accepted. * @param {boolean} [verbose=false] To print debugging info * @returns {boolean} Whether the input string is accepted. */ _acceptString(inputStr: string, verbose: boolean = false): boolean { return this.handle.AcceptString(inputStr, verbose); } /** * Returns a bitmask in the form of an Int32Array of length ceildiv(vocab_size, 32) * based on what tokens can/cannot be accepted by the current state of the grammar state matcher. * * @returns {Int32Array} An array representing the bitmask that masks the rejected token IDs */ async getNextTokenBitmask(): Promise { await asyncInitBinding(); // a handle of std::vector const maskIntVector = this.handle.GetNextTokenBitmask(this.vocab_size) const maskInt32Array = binding.vecIntToView(maskIntVector).slice(); maskIntVector.delete(); return maskInt32Array; } /** * Check if the matcher has accepted the stop token and terminated. See also * GrammarMatcher.acceptToken. */ isTerminated(): boolean { return this.handle.IsTerminated(); } /** * Check if the input accepted so far forms a complete valid string according to the grammar. * Unlike isTerminated(), this does not require the stop token to have been accepted. */ isCompleted(): boolean { return this.handle.IsCompleted(); } /** * Reset the matcher to the initial state. */ reset(): void { this.handle.Reset(); } /** * Find the jump-forward string for jump-forward decoding. This is the longest string that * will be valid according to the current syntax. * @returns {string} The jump-forward string. */ findJumpForwardString(): string { return this.handle.FindJumpForwardString(); } /** * Rollback the matcher to a previous state. * @param {number} numTokens The number of tokens to rollback. It cannot exceed the current * number of steps, nor can it exceed the specified maximum number of rollback tokens. */ rollBack(numTokens: number): void { this.handle.Rollback(numTokens); } } xgrammar-0.2.3/web/src/xgrammar_binding.cc000066400000000000000000000212211521764210300205140ustar00rootroot00000000000000/* * \file xgrammar_binding.cc * \brief XGrammar wasm runtime library pack. */ // Configuration for XGRAMMAR_LOG() #define XGRAMMAR_LOG_CUSTOMIZE 1 #include #include #include #include #include #include "../../cpp/json_schema_converter.h" #include "../../cpp/support/utils.h" #include "../../cpp/testing.h" namespace xgrammar { // Override logging mechanism [[noreturn]] void LogFatalImpl(const std::string& file, int lineno, const std::string& message) { std::cerr << "[FATAL] " << file << ":" << lineno << ": " << message << std::endl; abort(); } void LogMessageImpl(const std::string& file, int lineno, int level, const std::string& message) { static const char* level_strings_[] = { "[INFO] ", "[DEBUG] ", "[WARNING] ", }; std::cout << level_strings_[level] << file << ":" << lineno << ": " << message << std::endl; } /*! * \brief Helper that unwraps Grammar::FromStructuralTag by throwing variant errors. */ Grammar Grammar_FromStructuralTag(const std::string& structural_tag_json) { auto result = Grammar::FromStructuralTag(structural_tag_json, std::nullopt); if (std::holds_alternative(result)) { ThrowVariantError(std::get(result)); } return std::get(result); } /*! * \brief Wrapper for GrammarCompiler::CompileStructuralTag that validates the structural tag JSON. */ CompiledGrammar GrammarCompiler_CompileStructuralTag( GrammarCompiler& compiler, const std::string& structural_tag_json ) { auto result = Grammar::FromStructuralTag(structural_tag_json, std::nullopt); if (std::holds_alternative(result)) { ThrowVariantError(std::get(result)); } return compiler.CompileStructuralTag(structural_tag_json); } } // namespace xgrammar using namespace emscripten; using namespace xgrammar; TokenizerInfo TokenizerInfo_Init( const std::vector& encoded_vocab, std::string vocab_type, std::optional vocab_size, std::optional> stop_token_ids, bool add_prefix_space ) { static const std::unordered_map VOCAB_TYPE_MAP = { {"RAW", VocabType::RAW}, {"BYTE_FALLBACK", VocabType::BYTE_FALLBACK}, {"BYTE_LEVEL", VocabType::BYTE_LEVEL}, }; return TokenizerInfo( encoded_vocab, VOCAB_TYPE_MAP.at(vocab_type), vocab_size, stop_token_ids, add_prefix_space ); } GrammarMatcher GrammarMatcher_Init( const CompiledGrammar& grammar, std::optional> override_stop_tokens, bool terminate_without_stop_token, int max_rollback_tokens ) { return GrammarMatcher( grammar, override_stop_tokens, terminate_without_stop_token, max_rollback_tokens ); } /*! * \brief Finds the next token bitmask of the matcher. */ std::vector GrammarMatcher_GetNextTokenBitmask(GrammarMatcher& matcher, int vocab_size) { // 1. Initialize std::vector result auto buffer_size = GetBitmaskSize(vocab_size); std::vector result(buffer_size); // 2. Initialize DLTensor with the data pointer of the std vector. DLTensor tensor; tensor.data = result.data(); tensor.device = DLDevice{kDLCPU, 0}; tensor.ndim = 1; tensor.dtype = DLDataType{kDLInt, 32, 1}; // int32 std::vector shape = {buffer_size}; tensor.shape = &shape[0]; std::vector strides = {1}; tensor.strides = &strides[0]; tensor.byte_offset = 0; // 3. Populate tensor, hence result matcher.FillNextTokenBitmask(&tensor); return result; } /*! * \brief Return the list of rejected token IDs based on the bit mask. * \note This method is mainly used in testing, so performance is not as important. */ std::vector Testing_DebugGetMaskedTokensFromBitmask( std::vector token_bitmask, size_t vocab_size, int index ) { // 1. Convert token_bitmask into DLTensor DLTensor tensor; tensor.data = token_bitmask.data(); tensor.device = DLDevice{kDLCPU, 0}; tensor.ndim = 1; tensor.dtype = DLDataType{kDLInt, 32, 1}; // int32 std::vector shape = {static_cast(token_bitmask.size())}; tensor.shape = &shape[0]; std::vector strides = {1}; tensor.strides = &strides[0]; tensor.byte_offset = 0; // 2. Get rejected token IDs std::vector result; _DebugGetMaskedTokensFromBitmask(&result, tensor, vocab_size, index); return result; } /*! * \brief Helps view an std::vector handle as Int32Array in JS without copying. */ emscripten::val vecIntToView(const std::vector& vec) { return emscripten::val(typed_memory_view(vec.size(), vec.data())); } std::vector VecStringFromJSArray(const emscripten::val& js_array) { return emscripten::vecFromJSArray(js_array); } std::vector VecIntFromJSArray(const emscripten::val& js_array) { return emscripten::vecFromJSArray(js_array); } EMSCRIPTEN_BINDINGS(xgrammar) { enum_("JSONFormat") .value("kJSON", xgrammar::JSONFormat::kJSON) .value("kQwenXML", xgrammar::JSONFormat::kQwenXML) .value("kMiniMaxXML", xgrammar::JSONFormat::kMiniMaxXML) .value("kDeepSeekXML", xgrammar::JSONFormat::kDeepSeekXML); // Register std::optional used in Grammar::FromJSONSchema register_optional(); value_object>("StringPair") .field("first", &std::pair::first) .field("second", &std::pair::second); register_optional>(); // Register std::vector for TokenizerInfo.GetDecodedVocab() register_vector("VectorString"); function( "vecStringFromJSArray", select_overload(const emscripten::val&)>(&VecStringFromJSArray) ); // Register std::optional> for GrammarMatcher_Init register_vector("VectorInt"); register_optional>(); function( "vecIntFromJSArray", select_overload(const emscripten::val&)>(&VecIntFromJSArray) ); // Register view so we can read std::vector as Int32Array in JS without copying function("vecIntToView", &vecIntToView); // Testing methods function( "_JSONSchemaToEBNF", select_overload, std::optional>, bool, std::optional, JSONFormat, bool )>(&JSONSchemaToEBNF) ); function("DebugGetMaskedTokensFromBitmask", &Testing_DebugGetMaskedTokensFromBitmask); function("EBNFToGrammarNoNormalization", &_EBNFToGrammarNoNormalization); class_("Grammar") .function("ToString", &Grammar::ToString) .class_function("FromEBNF", &Grammar::FromEBNF) .class_function("FromJSONSchema", &Grammar::FromJSONSchema) .class_function("BuiltinJSONGrammar", &Grammar::BuiltinJSONGrammar) .class_function("FromStructuralTag", &Grammar_FromStructuralTag); class_("TokenizerInfo") .constructor(&TokenizerInfo_Init) .function("GetVocabSize", &TokenizerInfo::GetVocabSize) .function("GetDecodedVocab", &TokenizerInfo::GetDecodedVocab); class_("CompiledGrammar") .function("GetGrammar", &CompiledGrammar::GetGrammar) .function("GetTokenizerInfo", &CompiledGrammar::GetTokenizerInfo); class_("GrammarCompiler") .constructor() .function("CompileJSONSchema", &GrammarCompiler::CompileJSONSchema) .function("CompileBuiltinJSONGrammar", &GrammarCompiler::CompileBuiltinJSONGrammar) .function( "CompileGrammar", select_overload(&GrammarCompiler::CompileGrammar) ) .function("CompileStructuralTag", &GrammarCompiler_CompileStructuralTag) .function("ClearCache", &GrammarCompiler::ClearCache); class_("GrammarMatcher") .constructor(&GrammarMatcher_Init) .smart_ptr>("GrammarMatcher") .function("GetMaxRollbackTokens", &GrammarMatcher::GetMaxRollbackTokens) .function("AcceptToken", &GrammarMatcher::AcceptToken) .function("GetNextTokenBitmask", &GrammarMatcher_GetNextTokenBitmask) .function("IsTerminated", &GrammarMatcher::IsTerminated) .function("IsCompleted", &GrammarMatcher::IsCompleted) .function("Reset", &GrammarMatcher::Reset) .function("FindJumpForwardString", &GrammarMatcher::FindJumpForwardString) .function("Rollback", &GrammarMatcher::Rollback) .function("AcceptString", &GrammarMatcher::AcceptString); } xgrammar-0.2.3/web/tests/000077500000000000000000000000001521764210300152525ustar00rootroot00000000000000xgrammar-0.2.3/web/tests/grammar.test.ts000066400000000000000000000715211521764210300202340ustar00rootroot00000000000000/** * Test all APIs exposed in the web-xgrammar package. The goal of these unit tests * are to test each API works as expected. It does not test behavior correctness * thoroughly since that is done in `tests/python`. */ import { describe, expect, test } from "@jest/globals"; import { Grammar, GrammarCompiler, TokenizerInfo, GrammarMatcher, Testings, StructuralTagItem } from ".."; import { Tokenizer } from "@mlc-ai/web-tokenizers"; async function getTokenizerInfoFromUrl(tokenizerUrl: string, vocabType: string, prependSpace: boolean): Promise { // 1. Get tokenizer const jsonBuffer = await (await fetch(tokenizerUrl)).arrayBuffer(); const tokenizer = await Tokenizer.fromJSON(jsonBuffer); // 2. Get raw vocab const encodedVocab: string[] = []; const vocabSize = tokenizer.getVocabSize(); for (let tokenId = 0; tokenId < vocabSize; tokenId++) { encodedVocab.push(tokenizer.idToToken(tokenId)); } // 3. Decode const decodedVocab = await TokenizerInfo.createTokenizerInfo(encodedVocab, vocabType, prependSpace); return decodedVocab; } describe("Test all Grammar APIs", () => { const ebnf_grammar = String.raw`basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^\0-\x1f\"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" root_prop_1 ::= basic_boolean | basic_null root_prop_2 ::= basic_number | basic_null root_part_2 ::= "" | [ \n\t]* "," [ \n\t]* "\"name\"" [ \n\t]* ":" [ \n\t]* basic_string "" root_part_1 ::= [ \n\t]* "," [ \n\t]* "\"size\"" [ \n\t]* ":" [ \n\t]* root_prop_2 root_part_2 root_part_0 ::= root_part_1 | [ \n\t]* "," [ \n\t]* "\"opt_bool\"" [ \n\t]* ":" [ \n\t]* root_prop_1 root_part_1 root ::= "{" [ \n\t]* (("\"num\"" [ \n\t]* ":" [ \n\t]* basic_integer root_part_0) | ("\"opt_bool\"" [ \n\t]* ":" [ \n\t]* root_prop_1 root_part_1) | ("\"size\"" [ \n\t]* ":" [ \n\t]* root_prop_2 root_part_2)) [ \n\t]* "}" `; /** * Equivalent to class MainModel(BaseModel): num: int = 0 opt_bool: Optional[bool] = None size: Optional[float] name: str = "" */ const schema = String.raw`{"properties": {"num": {"default": 0, "title": "Num", "type": "integer"}, "opt_bool": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "default": null, "title": "Opt Bool"}, "size": {"anyOf": [{"type": "number"}, {"type": "null"}], "title": "Size"}, "name": {"default": "", "title": "Name", "type": "string"}}, "required": ["size"], "title": "MainModel", "type": "object"}`; test("Test Grammar.fromEBNF and Grammar.toString", async () => { const before = `root ::= ((b c) | (b root)) b ::= ((b_1 d [a]*)) c ::= ((c_1)) d ::= ((d_1)) (=([a]*)) b_1 ::= ("" | ("b" b_1)) (=(d [a]*)) c_1 ::= (([acep-z] c_1) | ([acep-z])) d_1 ::= ("" | ("d")) `; const grammar1 = await Grammar.fromEBNF(before) const outputStr = grammar1.toString(); expect(outputStr).toEqual(before); }); test("Test Grammar.fromJSONSchema()", async () => { const grammar = await Grammar.fromJSONSchema(schema, false); const outputStr = grammar.toString(); expect(outputStr == "").toEqual(false); }); test("Test _jsonSchemaToEBNF", async () => { // Equivalent to test_optional() in test_json_schema_converter.py const grammar = await Testings._jsonSchemaToEBNF(schema, true, -1); expect(grammar).toEqual(ebnf_grammar); }); test("Test indent _jsonSchemaToEBNF", async () => { const grammar0 = await Testings._jsonSchemaToEBNF(schema, false, -1); const grammar1 = await Testings._jsonSchemaToEBNF(schema, false); const grammar2 = await Testings._jsonSchemaToEBNF(schema, false, 2); expect(grammar1).toEqual(grammar2); expect(grammar0).not.toEqual(grammar2); }); test("Test indent Grammar.fromJSONSchema()", async () => { const grammar0 = (await Grammar.fromJSONSchema(schema, false, -1)).toString(); const grammar1 = (await Grammar.fromJSONSchema(schema, false)).toString(); const grammar2 = (await Grammar.fromJSONSchema(schema, false, 2)).toString(); expect(grammar1).toEqual(grammar2); expect(grammar0).not.toEqual(grammar2); }); test("Test Grammar.builtinJSONGrammar()", async () => { const grammar = await Grammar.builtinJSONGrammar(); const outputStr = grammar.toString(); expect(outputStr == "").toEqual(false); }); test("Test Grammar.fromStructuralTag()", async () => { const schemaObject = { type: "object", properties: { city: { type: "string" }, unit: { enum: ["celsius", "fahrenheit"] }, }, required: ["city"], additionalProperties: false, }; const structuralTag = { type: "structural_tag" as const, format: { type: "triggered_tags" as const, triggers: ["", end: "", content: { type: "json_schema" as const, json_schema: schemaObject, }, }, ], }, }; const grammar = await Grammar.fromStructuralTag(structuralTag); const legacyGrammar = await Grammar.fromStructuralTag( [new StructuralTagItem("", JSON.stringify(schemaObject), "")], [" { test("Test basic tokenizer info", async () => { const dummyVocab = ["!", "éĶ¦"]; const dummyVocabType = "byte_level"; const tokenizerInfo = await TokenizerInfo.createTokenizerInfo( dummyVocab, dummyVocabType, false ); expect(tokenizerInfo.getDecodedVocabHandle().get(0)).toEqual("!"); expect(tokenizerInfo.getDecodedVocabHandle().get(1)).toEqual("锦"); tokenizerInfo.dispose(); }); test("Test with Llama3.2, byte_level", async () => { const tokenizerInfo = await getTokenizerInfoFromUrl( "https://huggingface.co/mlc-ai/Llama-3.2-1B-Instruct-q4f16_0-MLC/raw/main/tokenizer.json", "byte_level", false, ); expect(tokenizerInfo.getDecodedVocabHandle().size()).toEqual(128256); tokenizerInfo.dispose(); }) test("Test with Phi3.5, byte_fallback", async () => { const tokenizerInfo = await getTokenizerInfoFromUrl( "https://huggingface.co/mlc-ai/Phi-3.5-mini-instruct-q4f16_1-MLC/raw/main/tokenizer.json", "byte_fallback", true, ); // phi-3.5 though vocab size is 32064 in config.json, has 32011 actual vocab. The size of the // table (i.e. tokenizer.getVocabSize()) may be smaller than the `vocab_size` in config.json // (length of logits), see https://github.com/QwenLM/Qwen2/issues/147 and // https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/discussions/47. expect(tokenizerInfo.getDecodedVocabHandle().size()).toEqual(32011); tokenizerInfo.dispose(); }) }); describe("Test CompiledGrammar and GrammarCompiler", () => { /** * Equivalent to class MainModel(BaseModel): num: int = 0 opt_bool: Optional[bool] = None size: Optional[float] name: str = "" */ const schema = String.raw`{"properties": {"num": {"default": 0, "title": "Num", "type": "integer"}, "opt_bool": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "default": null, "title": "Opt Bool"}, "size": {"anyOf": [{"type": "number"}, {"type": "null"}], "title": "Size"}, "name": {"default": "", "title": "Name", "type": "string"}}, "required": ["size"], "title": "MainModel", "type": "object"}`; test("Test compileBuiltinJSONGrammar", async () => { const tokenizerInfo = await getTokenizerInfoFromUrl( "https://huggingface.co/mlc-ai/Llama-3.2-1B-Instruct-q4f16_0-MLC/raw/main/tokenizer.json", "byte_level", false, ); const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); const compiledGrammar = await compiler.compileBuiltinJSONGrammar(); const outputStr = compiledGrammar.grammar().toString(); expect(outputStr == "").toEqual(false); expect(compiledGrammar.tokenizerInfo().getDecodedVocabHandle().size()).toEqual(128256); tokenizerInfo.dispose(); compiledGrammar.dispose(); compiler.dispose(); }); test("Test compileJSONSchema", async () => { const tokenizerInfo = await getTokenizerInfoFromUrl( "https://huggingface.co/mlc-ai/Llama-3.2-1B-Instruct-q4f16_0-MLC/raw/main/tokenizer.json", "byte_level", false, ); const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); const compiledGrammar0 = await compiler.compileJSONSchema(schema, false, -1); const compiledGrammar1 = await compiler.compileJSONSchema(schema, false); const compiledGrammar2 = await compiler.compileJSONSchema(schema, false, 2); const outputStr0 = compiledGrammar0.grammar().toString(); const outputStr1 = compiledGrammar1.grammar().toString(); const outputStr2 = compiledGrammar2.grammar().toString(); expect(outputStr1).toEqual(outputStr2); expect(outputStr0).not.toEqual(outputStr2); expect(compiledGrammar1.tokenizerInfo().getDecodedVocabHandle().size()).toEqual(128256); tokenizerInfo.dispose(); compiledGrammar0.dispose(); compiledGrammar1.dispose(); compiledGrammar2.dispose(); compiler.dispose(); }); test("Test compileGrammar with a EBNF string and a Grammar", async () => { const before = `root ::= ((b c) | (b root)) b ::= ((b_1 d [a]*)) c ::= ((c_1)) d ::= ((d_1)) (=([a]*)) b_1 ::= ("" | ("b" b_1)) (=(d [a]*)) c_1 ::= (([acep-z] c_1) | ([acep-z])) d_1 ::= ("" | ("d")) `; const tokenizerInfo = await getTokenizerInfoFromUrl( "https://huggingface.co/mlc-ai/Llama-3.2-1B-Instruct-q4f16_0-MLC/raw/main/tokenizer.json", "byte_level", false, ); const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); const grammar = await Grammar.fromEBNF(before); const compiledGrammar1 = await compiler.compileGrammar(grammar); const compiledGrammar2 = await compiler.compileGrammar(before); const outputStr1 = compiledGrammar1.grammar().toString(); const outputStr2 = compiledGrammar2.grammar().toString(); expect(outputStr1).toEqual(before); expect(outputStr2).toEqual(before); expect(compiledGrammar1.tokenizerInfo().getDecodedVocabHandle().size()).toEqual(128256); expect(compiledGrammar2.tokenizerInfo().getDecodedVocabHandle().size()).toEqual(128256); tokenizerInfo.dispose(); compiledGrammar1.dispose(); compiledGrammar2.dispose(); compiler.dispose(); }); }); // Identical to tests in `test_grammar_matcher.py` describe("Test GrammarMatcher E2E", () => { const vocab = [ "", "", "a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", "\n", " ", '"a":true', ]; const input_splitted = ["{", '"', "abc", 'b"', ":", "6", ", ", " ", '"a":true', "}"]; const input_ids: number[] = []; input_splitted.forEach((input) => { input_ids.push(vocab.indexOf(input)); }); test("test_token_operations", async () => { // 1. Instantiate matcher const tokenizerInfo = await TokenizerInfo.createTokenizerInfo( vocab, "byte_level", false ); const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); const jsonGrammar = await compiler.compileBuiltinJSONGrammar(); const matcher = await GrammarMatcher.createGrammarMatcher(jsonGrammar); // 2. Test const expected = [ ["{"], ['"', "}", "\n", " ", '"a":true'], ["", "a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", " "], ["", "a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", " "], [':"', ":", "\n", " "], ['"', "{", "6", "\n", " "], ["}", ", ", "6", "\n", " "], ['"', "\n", " ", '"a":true'], ['"', "\n", " ", '"a":true'], ["}", ", ", "\n", " "], [""], ] const result: Array> = [] for (let i = 0; i <= input_ids.length; i++) { const input_id = input_ids[i]; // Find rejected IDs const bitmask = await matcher.getNextTokenBitmask(); const rejectedIDs = await Testings.debugGetMaskedTokensFromBitmask( bitmask, tokenizerInfo.getVocabSize() ); // Find accepted tokens const vocabIDSet = new Set([...Array(vocab.length).keys()]); const rejectedIDSet = new Set(rejectedIDs); const acceptedIDSet = new Set([...vocabIDSet].filter(x => !rejectedIDSet.has(x))); const acceptedIDList = Array.from(acceptedIDSet.values()).sort(((a, b) => a - b)); const acceptedTokens: string[] = []; acceptedIDList.forEach((acceptedID) => { acceptedTokens.push(vocab[acceptedID]); }); result.push(acceptedTokens); // Note the <= in the loop bound. We do an extra checking for the last input. if (i < input_ids.length) { // Check input_id is accepted, and update matcher expect(acceptedIDSet.has(input_id)).toEqual(true); const accepted = matcher.acceptToken(input_id); expect(accepted).toEqual(true); } } expect(result).toEqual(expected); matcher.dispose(); tokenizerInfo.dispose(); }); // Identical to the test above, except we specify stop token to be both 0 and 1 test("test_token_operations with customized stop token id", async () => { // 1. Instantiate matcher // TODO(Charlie): Specifying only 0 still makes 1 a valid stop token -- is this what we want? const tokenizerInfo = await TokenizerInfo.createTokenizerInfo( vocab, "byte_level", false, undefined, [0, 1] ); const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); const jsonGrammar = await compiler.compileBuiltinJSONGrammar(); const matcher = await GrammarMatcher.createGrammarMatcher(jsonGrammar); // 2. Test const expected = [ ["{"], ['"', "}", "\n", " ", '"a":true'], ["a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", " "], ["a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", " "], [':"', ":", "\n", " "], ['"', "{", "6", "\n", " "], ["}", ", ", "6", "\n", " "], ['"', "\n", " ", '"a":true'], ['"', "\n", " ", '"a":true'], ["}", ", ", "\n", " "], ["", ""], ] const result: Array> = [] for (let i = 0; i <= input_ids.length; i++) { const input_id = input_ids[i]; // Find rejected IDs const bitmask = await matcher.getNextTokenBitmask(); const rejectedIDs = await Testings.debugGetMaskedTokensFromBitmask( bitmask, tokenizerInfo.getVocabSize() ); // Find accepted tokens const vocabIDSet = new Set([...Array(vocab.length).keys()]); const rejectedIDSet = new Set(rejectedIDs); const acceptedIDSet = new Set([...vocabIDSet].filter(x => !rejectedIDSet.has(x))); const acceptedIDList = Array.from(acceptedIDSet.values()).sort(((a, b) => a - b)); const acceptedTokens: string[] = []; acceptedIDList.forEach((acceptedID) => { acceptedTokens.push(vocab[acceptedID]); }); result.push(acceptedTokens); // Note the <= in the loop bound. We do an extra checking for the last input. if (i < input_ids.length) { // Check input_id is accepted, and update matcher expect(acceptedIDSet.has(input_id)).toEqual(true); const accepted = matcher.acceptToken(input_id); expect(accepted).toEqual(true); } } expect(result).toEqual(expected); tokenizerInfo.dispose(); matcher.dispose(); }); test("test_roll_back", async () => { // 1. Instantiate matcher const tokenizerInfo = await TokenizerInfo.createTokenizerInfo( vocab, "byte_level", false ); const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); const jsonGrammar = await compiler.compileBuiltinJSONGrammar(); const matcher = await GrammarMatcher.createGrammarMatcher( jsonGrammar, undefined, undefined, 5, ); tokenizerInfo.dispose(); // Max rollback tokens is deprecated expect(matcher.getMaxRollbackTokens()).toEqual(-1); // 2. Test const input_ids_splitted: number[][] = []; for (let i = 0; i < input_ids.length; i += 2) { input_ids_splitted.push(input_ids.slice(i, i + 2)); } for (let i = 0; i < input_ids_splitted.length; i++) { const i_1 = input_ids_splitted[i][0]; const i_2 = input_ids_splitted[i][1]; const orig_result: Int32Array[] = []; // Accept firt round orig_result.push(await matcher.getNextTokenBitmask()); const accept_i1 = matcher.acceptToken(i_1); orig_result.push(await matcher.getNextTokenBitmask()); const accept_i2 = matcher.acceptToken(i_2); expect(accept_i1).toEqual(true); expect(accept_i2).toEqual(true); // Rollback, then accept again matcher.rollBack(2); const result_after_rollback: Int32Array[] = []; result_after_rollback.push(await matcher.getNextTokenBitmask()); const accept_i1_r = matcher.acceptToken(i_1); result_after_rollback.push(await matcher.getNextTokenBitmask()); const accept_i2_r = matcher.acceptToken(i_2); expect(accept_i1_r).toEqual(true); expect(accept_i2_r).toEqual(true); // Expect same token bitmask expect(orig_result).toEqual(result_after_rollback); } matcher.dispose(); }); test("test reset and termination", async () => { // This one has ``, different from the ones used before const vocab = [ "", "", "a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", "\n", " ", '"a":true', ]; const input_splitted = ["{", '"', "abc", 'b"', ":", "6", ", ", " ", '"a":true', "}", ""]; const input_ids: number[] = []; input_splitted.forEach((input) => { input_ids.push(vocab.indexOf(input)); }); // 1. Instantiate matcher const tokenizerInfo = await TokenizerInfo.createTokenizerInfo( vocab, "byte_level", false ); const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); const jsonGrammar = await compiler.compileBuiltinJSONGrammar(); const matcher = await GrammarMatcher.createGrammarMatcher( jsonGrammar, undefined, undefined, 5 ); tokenizerInfo.dispose(); // 2. Accept all one time const orig_result: Int32Array[] = []; for (let i = 0; i < input_ids.length; i++) { orig_result.push(await matcher.getNextTokenBitmask()); const accepted = matcher.acceptToken(input_ids[i]); expect(accepted).toEqual(true); } // 3. Check termination expect(matcher.isTerminated()).toEqual(true); const acceptedAfterTerm0 = matcher.acceptToken(0); expect(acceptedAfterTerm0).toEqual(false); // this will throw error, but cannot be caught by jest // await matcher.getNextTokenBitmask() // 4. Reset, accept again matcher.reset(); const result_after_reset: Int32Array[] = []; for (let i = 0; i < input_ids.length; i++) { result_after_reset.push(await matcher.getNextTokenBitmask()); const accepted = matcher.acceptToken(input_ids[i]); expect(accepted).toEqual(true); } // 5. Check same bitmask result, check termination again expect(orig_result).toEqual(result_after_reset); expect(matcher.isTerminated()).toEqual(true); const acceptedAfterTerm1 = matcher.acceptToken(0); expect(acceptedAfterTerm1).toEqual(false); // this will throw error, but cannot be caught by jest // await matcher.getNextTokenBitmask() // 6. Rollback 2, and should not be terminated and should accept "}" matcher.rollBack(2); expect(matcher.isTerminated()).toEqual(false); const acceptedAfterTerm2 = matcher.acceptToken(input_ids.slice(-2, -1)[0]); expect(acceptedAfterTerm2).toEqual(true); matcher.dispose(); }); test("test isCompleted", async () => { // Same vocab as "test reset and termination", with as stop token (index 1) const vocab = [ "", "", "a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", "\n", " ", '"a":true', ]; // Complete JSON object tokens *without* the stop token const input_without_stop = ["{", '"', "abc", 'b"', ":", "6", ", ", " ", '"a":true', "}"]; const input_ids_without_stop: number[] = []; input_without_stop.forEach((input) => { input_ids_without_stop.push(vocab.indexOf(input)); }); const stop_token_id = vocab.indexOf(""); // 1. Instantiate matcher (default: terminate_without_stop_token = false) const tokenizerInfo = await TokenizerInfo.createTokenizerInfo( vocab, "byte_level", false ); const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); const jsonGrammar = await compiler.compileBuiltinJSONGrammar(); const matcher = await GrammarMatcher.createGrammarMatcher( jsonGrammar, undefined, undefined, 5 ); tokenizerInfo.dispose(); // 2. Before input: not completed, not terminated expect(matcher.isCompleted()).toEqual(false); expect(matcher.isTerminated()).toEqual(false); // 3. Feed complete JSON object (no stop token) for (let i = 0; i < input_ids_without_stop.length; i++) { const accepted = matcher.acceptToken(input_ids_without_stop[i]); expect(accepted).toEqual(true); } // 4. Completed but not terminated expect(matcher.isCompleted()).toEqual(true); expect(matcher.isTerminated()).toEqual(false); // 5. Accept stop token: both completed and terminated const acceptStop = matcher.acceptToken(stop_token_id); expect(acceptStop).toEqual(true); expect(matcher.isCompleted()).toEqual(true); expect(matcher.isTerminated()).toEqual(true); // 6. Rollback stop token: still completed, not terminated matcher.rollBack(1); expect(matcher.isCompleted()).toEqual(true); expect(matcher.isTerminated()).toEqual(false); // 7. Rollback further into mid-parse: neither completed nor terminated matcher.rollBack(2); expect(matcher.isCompleted()).toEqual(false); expect(matcher.isTerminated()).toEqual(false); matcher.dispose(); }); test("test_get_jump_forward_string", async () => { const grammar_ebnf = String.raw`root ::= "abb" | "abbd" | other_rule other_rule ::= "a" sub_rule "b" sub_rule ::= "b" `; const vocab = ["a", "bb"]; const tokenizerInfo = await TokenizerInfo.createTokenizerInfo( vocab, "byte_level", false ); const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); const grammar = await compiler.compileGrammar(grammar_ebnf); const matcher = await GrammarMatcher.createGrammarMatcher(grammar); tokenizerInfo.dispose(); expect(matcher.acceptToken(0)).toEqual(true); expect(matcher.findJumpForwardString()).toEqual("bb"); }); }); // Identical to `test_builtin_grammar_json_schema.py` describe("Test json schema E2E", () => { // Equivalent to MainModel used in the python test, except removed minItem and maxItem from tuple_field to avoid long log const schemaStr = String.raw`{"properties": {"integer_field": {"title": "Integer Field", "type": "integer"}, "number_field": {"title": "Number Field", "type": "number"}, "boolean_field": {"title": "Boolean Field", "type": "boolean"}, "any_array_field": {"items": {}, "title": "Any Array Field", "type": "array"}, "array_field": {"items": {"type": "string"}, "title": "Array Field", "type": "array"}, "tuple_field": {"prefixItems": [{"type": "string"}, {"type": "integer"}, {"items": {"type": "string"}, "type": "array"}], "title": "Tuple Field", "type": "array"}, "object_field": {"additionalProperties": {"type": "integer"}, "title": "Object Field", "type": "object"}, "nested_object_field": {"additionalProperties": {"additionalProperties": {"type": "integer"}, "type": "object"}, "title": "Nested Object Field", "type": "object"}}, "required": ["integer_field", "number_field", "boolean_field", "any_array_field", "array_field", "tuple_field", "object_field", "nested_object_field"], "title": "MainModel", "type": "object"}`; // Equivalent to instance in the python test, a valid json following the above schema const instanceStr = String.raw`{ "integer_field": 42, "number_field": 314000.0, "boolean_field": true, "any_array_field": [ 3.14, "foo", null, true ], "array_field": [ "foo", "bar" ], "tuple_field": [ "foo", 42, [ "bar", "baz" ] ], "object_field": { "foo": 42, "bar": 43 }, "nested_object_field": { "foo": { "bar": 42 } } }`; // Note: This test much slower than others test("Test with Llama3.2, byte_level", async () => { // 1. Get tokenizer const jsonBuffer = await (await fetch( "https://huggingface.co/mlc-ai/Llama-3.2-1B-Instruct-q4f16_0-MLC/raw/main/tokenizer.json" )).arrayBuffer(); const tokenizer = await Tokenizer.fromJSON(jsonBuffer); // 2. Get encoded vocab const encodedVocab: string[] = []; const vocabSize = tokenizer.getVocabSize(); for (let tokenId = 0; tokenId < vocabSize; tokenId++) { encodedVocab.push(tokenizer.idToToken(tokenId)); } // 3. Decode const tokenizerInfo = await TokenizerInfo.createTokenizerInfo(encodedVocab, "byte_level", false); const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); // 4. Instantiate matcher const grammar = await compiler.compileJSONSchema(schemaStr, true, 2); const matcher = await GrammarMatcher.createGrammarMatcher(grammar); const inputIds = tokenizer.encode(instanceStr); // 5. Expect to accept all inputIds for (let i = 0; i < inputIds.length; i++) { const inputId = inputIds[i]; await matcher.getNextTokenBitmask(); const accepted = matcher.acceptToken(inputId); expect(accepted).toEqual(true); } // 6. Check finalization const final_bitmask = await matcher.getNextTokenBitmask(); expect(final_bitmask.length).toEqual(Math.ceil(128256 / 32)); const final_rejected_tokens = (await Testings.debugGetMaskedTokensFromBitmask( final_bitmask, tokenizerInfo.getVocabSize() )); expect(final_rejected_tokens.indexOf(128001)).toEqual(-1); // stop token not rejected const acceptStop = matcher.acceptToken(128001); expect(acceptStop).toEqual(true); expect(matcher.isTerminated()).toEqual(true); matcher.dispose(); grammar.dispose(); tokenizerInfo.dispose(); }); // Note: This test much slower than others test("Test with Phi-3.5, byte_fallback, _debugAcceptString", async () => { // 1. Get tokenizer const jsonBuffer = await (await fetch( "https://huggingface.co/mlc-ai/Phi-3.5-mini-instruct-q4f16_1-MLC/raw/main/tokenizer.json", )).arrayBuffer(); const tokenizer = await Tokenizer.fromJSON(jsonBuffer); // 2. Get encoded vocab const encodedVocab: string[] = []; const vocabSize = tokenizer.getVocabSize(); for (let tokenId = 0; tokenId < vocabSize; tokenId++) { encodedVocab.push(tokenizer.idToToken(tokenId)); } // 3. Decode; note that phi-3.5 has 32064 as vocab size in `config.json` const tokenizerInfo = await TokenizerInfo.createTokenizerInfo(encodedVocab, "byte_fallback", false, 32064); const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); // 4. Instantiate matcher const grammar = await compiler.compileJSONSchema(schemaStr, true, 2); const matcher = await GrammarMatcher.createGrammarMatcher(grammar); // 5. Expect to accept all inputIds for (let i = 0; i < instanceStr.length; i++) { const inputStr = instanceStr[i]; await matcher.getNextTokenBitmask(); // if use acceptToken, the first token of instanceStr will be encoded as 426, `_{`, // while the matcher only accepts 29912, `{`, and another token of `<0x??>` const accepted = matcher._acceptString(inputStr); expect(accepted).toEqual(true); } // 6. Check finalization const final_bitmask = await matcher.getNextTokenBitmask(); // Tests how phi3.5 has dummy padded tokens. See https://github.com/mlc-ai/mlc-llm/pull/2651 expect(final_bitmask.length).toEqual(Math.ceil(32064 / 32)); const final_rejected_tokens = (await Testings.debugGetMaskedTokensFromBitmask( final_bitmask, tokenizerInfo.getVocabSize() )); expect(final_rejected_tokens.indexOf(2)).toEqual(-1); // stop token not rejected expect(final_rejected_tokens.indexOf(32000)).toEqual(-1); // stop token not rejected const acceptStop = matcher.acceptToken(2); expect(acceptStop).toEqual(true); expect(matcher.isTerminated()).toEqual(true); tokenizerInfo.dispose(); }) }); xgrammar-0.2.3/web/tests/web_tokenizers_shim.mjs000066400000000000000000000003501521764210300220350ustar00rootroot00000000000000import { createRequire } from "module"; const require = createRequire(import.meta.url); const tokenizers = require("@mlc-ai/web-tokenizers/lib/index.cjs"); export const Tokenizer = tokenizers.Tokenizer; export default tokenizers; xgrammar-0.2.3/web/tsconfig.json000066400000000000000000000007031521764210300166170ustar00rootroot00000000000000{ "compilerOptions": { "target": "es2022", "declaration": true, "outDir": "lib", "declarationMap": true, "esModuleInterop": true, "strict": true, "noImplicitAny": false, // to allow importing xgrammar_binding.js "module": "esnext", "moduleResolution": "node", "lib": ["dom", "es2022"] }, "include": [ "src" ], "exclude": [ "node_modules", "build", "dist", "rollup.config.js" ] }