pax_global_header00006660000000000000000000000064152010256710014511gustar00rootroot0000000000000052 comment=0f69a3964be4d482fdab59ff393237afde999e7e zxc-0.11.0/000077500000000000000000000000001520102567100123745ustar00rootroot00000000000000zxc-0.11.0/.clang-format000066400000000000000000000003051520102567100147450ustar00rootroot00000000000000--- Language: Cpp BasedOnStyle: Google IndentWidth: 4 TabWidth: 4 UseTab: Never ColumnLimit: 100 AlignConsecutiveAssignments: false AlignConsecutiveDeclarations: false BreakBeforeBraces: Attachzxc-0.11.0/.clusterfuzzlite/000077500000000000000000000000001520102567100157305ustar00rootroot00000000000000zxc-0.11.0/.clusterfuzzlite/Dockerfile000066400000000000000000000005071520102567100177240ustar00rootroot00000000000000# Copyright (c) 2025-2026, Bertrand Lebonnois # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. FROM gcr.io/oss-fuzz-base/base-builder:v1 COPY . $SRC/zxc WORKDIR $SRC/zxc COPY .clusterfuzzlite/build.sh $SRC/build.shzxc-0.11.0/.clusterfuzzlite/build.sh000066400000000000000000000016051520102567100173650ustar00rootroot00000000000000#!/bin/bash -eu # Copyright (c) 2025-2026, Bertrand Lebonnois # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. AVAILABLE_FUZZERS="decompress roundtrip seekable pstream" LIB_SOURCES="src/lib/zxc_common.c src/lib/zxc_compress.c src/lib/zxc_decompress.c src/lib/zxc_driver.c src/lib/zxc_dispatch.c src/lib/zxc_huffman.c src/lib/zxc_seekable.c src/lib/zxc_pstream.c" for fuzzer in $AVAILABLE_FUZZERS; do if [ -z "${FUZZER_TARGET:-}" ] || [ "${FUZZER_TARGET}" == "$fuzzer" ]; then $CC $CFLAGS -I include \ -I src/lib/vendors \ -DZXC_FUNCTION_SUFFIX=_default -DZXC_ONLY_DEFAULT \ $LIB_SOURCES \ tests/fuzz_${fuzzer}.c \ -o $OUT/zxc_fuzzer_${fuzzer} \ $LIB_FUZZING_ENGINE \ -lm -pthread fi donezxc-0.11.0/.github/000077500000000000000000000000001520102567100137345ustar00rootroot00000000000000zxc-0.11.0/.github/CODEOWNERS000066400000000000000000000000751520102567100153310ustar00rootroot00000000000000# Default owners for everything in the repo * @hellobertrand zxc-0.11.0/.github/CODE_OF_CONDUCT.md000066400000000000000000000054601520102567100165400ustar00rootroot00000000000000# Contributor Covenant Code of Conduct ## Our Pledge We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex, gender characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. ## Our Standards Examples of behavior that contributes to a positive environment for our community include: * Demonstrating empathy and kindness toward other people * Being respectful of differing opinions, viewpoints, and experiences * Giving and gracefully accepting constructive feedback * Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience * Focusing on what is best not just for us as individuals, but for the overall community Examples of unacceptable behavior include: * The use of sexualized language or imagery, and sexual attention or advances of any kind * Trolling, insulting or derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or email address, without their explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Enforcement Responsibilities Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. ## Scope This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at **zxc.codec@gmail.com**. All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the reporter of any incident. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.1, available at [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. [homepage]: https://www.contributor-covenant.org [v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.htmlzxc-0.11.0/.github/CONTRIBUTING.md000066400000000000000000000027741520102567100161770ustar00rootroot00000000000000# Contributing to ZXC Thank you for your interest in contributing to ZXC! This guide will help you get started. ## Developer Certificate of Origin (DCO) By contributing, you certify that: - You have the right to submit the contribution - You agree to license your contribution under the BSD-3-Clause license Add this to your commits: ```bash git commit -s -m "Your commit message" ``` ## License Headers To maintain legal clarity and recognize all contributors, every new source file (.c, .h, .rs, .py, etc.) must include the following header at the very top: ```C /* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ ``` ## Quick Start ### Build and Test ```bash git clone https://github.com/hellobertrand/zxc.git cd zxc mkdir build && cd build cmake .. -DCMAKE_BUILD_TYPE=Debug make -j ctest --output-on-failure ``` ### Format Code ```bash clang-format -i src/lib/*.c include/*.h ``` ## Requirements - **C17** compiler (GCC, Clang, or MSVC) - **CMake** 3.10+ - Follow `.clang-format` style (Google) - All code must be ASCII-only - Pass `ctest` and static analysis ## Submitting Changes 1. Fork and create a feature branch 2. Add tests for new functionality 3. Ensure CI passes (build, tests, benchmarks) 4. Sign your commits with `-s` 5. Open a PR to `main` ## Reporting Issues Include: - ZXC version (`zxc --version`) - OS and architecture - Minimal reproduction steps Thank you for making ZXC better! zxc-0.11.0/.github/FUNDING.yml000066400000000000000000000000261520102567100155470ustar00rootroot00000000000000github: hellobertrand zxc-0.11.0/.github/SECURITY.md000066400000000000000000000004111520102567100155210ustar00rootroot00000000000000# Security Policy ## Reporting a Vulnerability **Please do not report security vulnerabilities through public GitHub issues.** If you have discovered a security issue in ZXC, please email us privately at **zxc.codec@gmail.com**. We will respond within 48 hours.zxc-0.11.0/.github/codeql/000077500000000000000000000000001520102567100152035ustar00rootroot00000000000000zxc-0.11.0/.github/codeql/codeql-config.yml000066400000000000000000000001601520102567100204350ustar00rootroot00000000000000name: "CodeQL Config" paths-ignore: - 'src/lib/vendors/rapidhash.h' queries: - uses: security-and-quality zxc-0.11.0/.github/dependabot.yml000066400000000000000000000007631520102567100165720ustar00rootroot00000000000000version: 2 updates: - package-ecosystem: github-actions directory: / schedule: interval: monthly - package-ecosystem: pip directory: /wrappers/python schedule: interval: monthly - package-ecosystem: cargo directory: /wrappers/rust schedule: interval: monthly - package-ecosystem: npm directory: /wrappers/nodejs schedule: interval: monthly - package-ecosystem: gomod directory: /wrappers/go schedule: interval: monthlyzxc-0.11.0/.github/workflows/000077500000000000000000000000001520102567100157715ustar00rootroot00000000000000zxc-0.11.0/.github/workflows/README.md000066400000000000000000000100441520102567100172470ustar00rootroot00000000000000# GitHub Actions Workflows This directory contains CI/CD workflows for the ZXC compression library. ## Core Workflows ### build.yml - Build & Release **Triggers:** Push to main, tags, pull requests, manual dispatch Builds and tests ZXC across multiple platforms (Linux x86_64/ARM64, macOS ARM64, Windows x64). Generates release artifacts and uploads binaries when tags are pushed. ### multiarch.yml - Multi-Architecture Build **Triggers:** Push to main, pull requests, manual dispatch Comprehensive build matrix testing across multiple architectures including 32-bit and 64-bit variants for Linux (x64, x86, ARM64, ARM) and Windows (x64, x86). Validates compilation compatibility across different platforms. ### multicomp.yml - Compiler Compatibility **Triggers:** Push to main, pull requests, manual dispatch Tests the codebase against a wide range of compilers (various versions of GCC and Clang) to ensure compatibility and identify any compiler-specific issues or warnings. ### benchmark.yml - Performance Benchmark **Triggers:** Push to main (src changes), pull requests, manual dispatch Runs performance benchmarks using LZbench on Ubuntu and macOS. Integrates ZXC into the LZbench framework and tests compression/decompression performance against the Silesia corpus. ## Quality & Security ### coverage.yml - Code Coverage **Triggers:** Push to main, pull requests, manual dispatch Builds the project with coverage instrumentation (`-DZXC_ENABLE_COVERAGE=ON`), runs unit and CLI tests, and generates a coverage report using `lcov`. The report is then uploaded to Codecov for analysis. ### fuzzing.yml - Fuzz Testing **Triggers:** Pull requests, scheduled (every 3 days), manual dispatch Executes fuzz testing using ClusterFuzzLite with multiple sanitizers (address, undefined) on decompression and roundtrip fuzzers. Helps identify memory safety issues and edge cases. ### quality.yml - Code Quality **Triggers:** Push to main, pull requests, manual dispatch Performs static analysis using Cppcheck and Clang Static Analyzer. Runs memory leak detection with Valgrind to ensure code quality and identify potential bugs. ### security.yml - Code Security **Triggers:** Push to main, pull requests Runs CodeQL security analysis to detect potential security vulnerabilities and coding errors in the C/C++ codebase. ### vendors.yml - Vendor Maintenance **Triggers:** Scheduled (weekly), manual dispatch Automatically checks for and updates third-party dependencies (like `rapidhash.h`) to ensure the project uses the latest stable versions of its vendors. ## Language Bindings ### wrapper-rust-publish.yml - Publish Rust Crates **Triggers:** Release published, manual dispatch Tests and publishes Rust crates to crates.io. Verifies the version matches the release tag, runs tests across platforms, and publishes `zxc-compress-sys` (FFI bindings) followed by `zxc-compress` (safe wrapper). ### wrapper-python-publish.yml - Publish Python Package **Triggers:** Release published, manual dispatch Builds platform-specific wheels using `cibuildwheel` for Linux (x86_64, ARM64), macOS (ARM64, Intel), and Windows (AMD64, ARM64). Tests wheels against Python 3.12-3.13, then publishes to PyPI via trusted publishing. ### wrapper-wasm.yml - WASM Build & Test **Triggers:** Release published, publish on main, manual dispatch Builds the WebAssembly target using Emscripten SDK. Compiles the library with SIMD disabled (scalar codepath) and no threading, then runs a Node.js roundtrip test suite covering all compression levels, reusable contexts, and error handling. Uploads `zxc.js` + `zxc.wasm` as build artifacts. ### wrapper-nodejs-publish.yml - Publish Node.js Package **Triggers:** Release published, manual dispatch Builds and publishes the Node.js package to npm. Handles the compilation of native bindings and ensures the package is correctly versioned and distributed. ### wrapper-go-test.yml - Test Go Package **Triggers:** Release published, manual dispatch Runs comprehensive tests for the Go bindings across various platforms and architectures to ensure the Go package is stable and functional. zxc-0.11.0/.github/workflows/benchmark.yml000066400000000000000000000103171520102567100204500ustar00rootroot00000000000000name: Benchmark on: workflow_dispatch: push: branches: [ main ] paths: - 'src/**' - '.github/workflows/benchmark.yml' pull_request: branches: [ main ] permissions: contents: read concurrency: cancel-in-progress: true group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} jobs: lzbench-benchmark: name: Run Benchmark if: github.event.pull_request.draft == false runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-latest, macos-26-xlarge] env: LZBENCH_DIR: lzbench SILESIA_TAR: silesia.tar steps: - name: Checkout Repository uses: actions/checkout@v6 - name: Select Latest Xcode if: runner.os == 'macOS' run: | sudo xcode-select -s /Applications/Xcode_26.4.app clang --version - name: Clone LZbench run: | # git clone --depth 1 https://github.com/inikep/lzbench "${LZBENCH_DIR}" git clone -b zxc-0.11.x https://github.com/hellobertrand/lzbench "${LZBENCH_DIR}" - name: Copy Lib ZXC run: | mkdir -p ${LZBENCH_DIR}/lz/zxc/include/ mkdir -p ${LZBENCH_DIR}/lz/zxc/src/lib/ cp -r include ${LZBENCH_DIR}/lz/zxc/ cp -r src/lib ${LZBENCH_DIR}/lz/zxc/src/ - name: Build Lzbench working-directory: ${{ env.LZBENCH_DIR }} run: | SKIP_CODECS="DONT_BUILD_BRIEFLZ=1 DONT_BUILD_BROTLI=1 DONT_BUILD_BSC=1" SKIP_CODECS+=" DONT_BUILD_BZIP2=1 DONT_BUILD_BZIP3=1 DONT_BUILD_CSC=1" SKIP_CODECS+=" DONT_BUILD_CRUSH=1 DONT_BUILD_DENSITY=1 DONT_BUILD_FASTLZ=1" SKIP_CODECS+=" DONT_BUILD_FASTLZMA2=1 DONT_BUILD_GIPFELI=1 DONT_BUILD_GLZA=1" SKIP_CODECS+=" DONT_BUILD_KANZI=1 DONT_BUILD_LIBDEFLATE=1 DONT_BUILD_LIZARD=1" SKIP_CODECS+=" DONT_BUILD_LZF=1 DONT_BUILD_LZFSE=1 DONT_BUILD_LZG=1" SKIP_CODECS+=" DONT_BUILD_LZHAM=1 DONT_BUILD_LZJB=1 DONT_BUILD_LZLIB=1" SKIP_CODECS+=" DONT_BUILD_LZMA=1 DONT_BUILD_LZMAT=1 DONT_BUILD_LZO=1" SKIP_CODECS+=" DONT_BUILD_LZRW=1 DONT_BUILD_LZSSE=1 DONT_BUILD_PPMD=1" SKIP_CODECS+=" DONT_BUILD_QUICKLZ=1 DONT_BUILD_SLZ=1 DONT_BUILD_TAMP=1" SKIP_CODECS+=" DONT_BUILD_TORNADO=1 DONT_BUILD_UCL=1 DONT_BUILD_WFLZ=1" SKIP_CODECS+=" DONT_BUILD_XZ=1 DONT_BUILD_YALZ77=1 DONT_BUILD_YAPPY=1" SKIP_CODECS+=" DONT_BUILD_ZLIB_NG=1 DONT_BUILD_ZLING=1 DONT_BUILD_ZPAQ=1" SKIP_CODECS+=" DONT_BUILD_MEMLZ=1" # Use latest available compiler per platform if [ "$RUNNER_OS" = "Linux" ]; then export CC=gcc-14 CXX=g++-14 fi make -j $SKIP_CODECS CC="${CC:-cc}" MOREFLAGS="-march=native" - name: Cache Silesia Corpus id: cache-silesia uses: actions/cache@v5 with: path: ${{ env.SILESIA_TAR }} key: silesia-tar-v1 - name: Download Silesia Corpus if: steps.cache-silesia.outputs.cache-hit != 'true' run: | wget -q https://github.com/DataCompression/corpus-collection/raw/main/Silesia-Corpus/silesia.tar.gz -O silesia.tar.gz gunzip silesia.tar.gz - name: SHA256 Checksum run: | shasum -a 256 "${SILESIA_TAR}" - name: Run Lzbench on Silesia Corpus working-directory: ${{ env.LZBENCH_DIR }} run: | if [ "${{ github.event_name }}" = "pull_request" ]; then CODECS="memcpy/zxc/lz4" else CODECS="memcpy/zxc/lz4/lz4fast,17/lz4hc,9/lzav,1/snappy/zstd_fast,-1/zstd,1/zlib,1" fi ./lzbench -e"${CODECS}" -j -t8,8 -o1 "../${SILESIA_TAR}" > ../benchmark.md echo "Benchmark finished. First lines:" head -n 40 ../benchmark.md - name: Add Benchmark to Job Summary run: | echo "### Benchmark Result (${{ matrix.os }})" >> $GITHUB_STEP_SUMMARY cat benchmark.md >> $GITHUB_STEP_SUMMARY - name: Upload Benchmark Result as Artifact uses: actions/upload-artifact@v7 with: name: benchmark-${{ matrix.os }}-${{ github.sha }} path: benchmark.md if-no-files-found: error retention-days: 10zxc-0.11.0/.github/workflows/build.yml000066400000000000000000000120341520102567100176130ustar00rootroot00000000000000name: Build & Release on: workflow_dispatch: push: branches: [ main ] tags: - 'v*' pull_request: branches: [ main ] permissions: contents: read concurrency: cancel-in-progress: true group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} jobs: build-and-test: name: Build ${{ matrix.name }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: - name: Linux x86_64 os: ubuntu-latest artifact_name: zxc asset_name: zxc-linux-x86_64 cmake_flags: "-DZXC_NATIVE_ARCH=OFF" - name: Linux ARM64 os: ubuntu-24.04-arm artifact_name: zxc asset_name: zxc-linux-aarch64 cmake_flags: "-DZXC_NATIVE_ARCH=OFF" - name: macOS ARM64 os: macos-26 artifact_name: zxc asset_name: zxc-macos-arm64 cmake_flags: "-DZXC_NATIVE_ARCH=OFF" - name: Windows x64 os: windows-latest artifact_name: zxc.exe asset_name: zxc-windows-x64 cmake_flags: "-A x64 -DZXC_NATIVE_ARCH=OFF" - name: Windows ARM64 os: windows-11-arm artifact_name: zxc.exe asset_name: zxc-windows-arm64 cmake_flags: "-A ARM64 -DZXC_NATIVE_ARCH=OFF" steps: - name: Checkout Repository uses: actions/checkout@v6 - name: Select Latest Compiler (Linux) if: runner.os == 'Linux' run: | echo "CC=gcc-14" >> $GITHUB_ENV echo "CXX=g++-14" >> $GITHUB_ENV gcc-14 --version - name: Select Latest Xcode (macOS) if: runner.os == 'macOS' run: | sudo xcode-select -s /Applications/Xcode_26.4.app clang --version - name: Configure CMake run: cmake -S . -B build -DCMAKE_BUILD_TYPE=Release ${{ matrix.cmake_flags }} - name: Build run: cmake --build build --config Release --parallel - name: Test (native) working-directory: build run: ctest -C Release --output-on-failure - name: Test CLI shell: bash run: | BUILD_DIR="build" # Find binary (handle both Ninja and Visual Studio generators) if [ -f "$BUILD_DIR/Release/${{ matrix.artifact_name }}" ]; then ZXC_BIN="$BUILD_DIR/Release/${{ matrix.artifact_name }}" elif [ -f "$BUILD_DIR/${{ matrix.artifact_name }}" ]; then ZXC_BIN="$BUILD_DIR/${{ matrix.artifact_name }}" else echo "Binary not found for CLI test!" find "$BUILD_DIR" -name "${{ matrix.artifact_name }}" exit 1 fi echo "Testing with binary: $ZXC_BIN" chmod +x tests/test_cli.sh ./tests/test_cli.sh "$ZXC_BIN" - name: Package Release (Archive) if: startsWith(github.ref, 'refs/tags/') shell: bash run: | # Install to staging directory using CMake (Standard way for all OS) cmake --install build --prefix release_staging --config Release # Strip binary on Unix-like systems only (installed to bin/) if [[ "${{ runner.os }}" != "Windows" ]]; then strip release_staging/bin/${{ matrix.artifact_name }} fi cp LICENSE release_staging/LICENSE.txt cp docs/man/zxc.1.md release_staging/MANUAL.md cat > release_staging/README.md <> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY lcov --summary coverage.info 2>&1 | tail -n +2 >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY - name: Upload Coverage Artifact uses: actions/upload-artifact@v7 with: name: coverage-report path: coverage-report retention-days: 10 - name: Upload coverage to Codecov uses: codecov/codecov-action@v6 with: token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage.info fail_ci_if_error: true zxc-0.11.0/.github/workflows/fuzzing.yml000066400000000000000000000173141520102567100202160ustar00rootroot00000000000000name: Fuzzing on: workflow_dispatch: inputs: mode: description: 'Fuzzing mode' required: true default: 'batch' type: choice options: - code-change - batch fuzz_seconds: description: 'Duration (seconds)' required: true default: '3600' pull_request: branches: [ main ] schedule: - cron: '0 1 * * 1' permissions: contents: read security-events: write jobs: fuzzing: name: Run Fuzzing (${{ matrix.fuzzer }} - ${{ matrix.sanitizer }}) runs-on: ubuntu-latest concurrency: group: ${{ github.workflow }}-${{ matrix.fuzzer }}-${{ matrix.sanitizer }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true strategy: fail-fast: false matrix: sanitizer: [address, undefined] fuzzer: [decompress, roundtrip, seekable, pstream] steps: - name: Checkout Repository uses: actions/checkout@v6 - name: Configure Fuzzer Target run: | sed -i '2i export FUZZER_TARGET="${{ matrix.fuzzer }}"' .clusterfuzzlite/build.sh # TODO: Remove this step once ClusterFuzzLite updates to support Docker 29+ - name: Downgrade Docker (Temporary Workaround) run: | # ClusterFuzzLite v1 uses Docker API 1.41 which is incompatible with Docker 29.0+ # Downgrade to Docker 28 until the action is updated curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt-get update # Install Docker 28.0.4 specifically sudo apt-get install -y --allow-downgrades docker-ce=5:28.0.4-1~ubuntu.$(lsb_release -rs)~$(lsb_release -cs) docker-ce-cli=5:28.0.4-1~ubuntu.$(lsb_release -rs)~$(lsb_release -cs) containerd.io sudo systemctl restart docker docker version - name: Build Fuzzers (${{ matrix.fuzzer }} - ${{ matrix.sanitizer }}) id: build uses: google/clusterfuzzlite/actions/build_fuzzers@v1 with: language: c github-token: ${{ secrets.GITHUB_TOKEN }} sanitizer: ${{ matrix.sanitizer }} - name: Run Fuzzers (${{ matrix.fuzzer }} - ${{ matrix.sanitizer }}) id: run uses: google/clusterfuzzlite/actions/run_fuzzers@v1 with: github-token: ${{ secrets.GITHUB_TOKEN }} mode: ${{ github.event_name == 'pull_request' && 'code-change' || inputs.mode || 'batch' }} fuzz-seconds: ${{ github.event_name == 'pull_request' && 120 || inputs.fuzz_seconds || 3600 }} sanitizer: ${{ matrix.sanitizer }} output-sarif: true storage-repo: https://${{ secrets.CFLITE_CORPUS_TOKEN }}@github.com/hellobertrand/zxc-fuzz-corpus.git storage-repo-branch: main - name: Upload SARIF to GitHub Security if: success() || failure() uses: github/codeql-action/upload-sarif@v4 with: sarif_file: . category: clusterfuzzlite-${{ matrix.fuzzer }}-${{ matrix.sanitizer }} tsan: name: Thread Sanitizer runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - name: Build with TSan run: | cmake -B build -DCMAKE_C_FLAGS="-fsanitize=thread -g -fno-omit-frame-pointer" -DCMAKE_BUILD_TYPE=Debug cmake --build build - name: Run Tests run: ctest --test-dir build --output-on-failure coverage: name: Fuzz Coverage Report runs-on: ubuntu-latest if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' steps: - uses: actions/checkout@v6 - name: Install LLVM Tools run: | sudo apt-get update -qq sudo apt-get install -y -qq llvm - name: Clone Fuzz Corpus run: | git clone --depth=1 \ https://${{ secrets.CFLITE_CORPUS_TOKEN }}@github.com/hellobertrand/zxc-fuzz-corpus.git \ corpus echo "roundtrip inputs: $(find corpus/corpus/zxc_fuzzer_roundtrip -type f | wc -l)" echo "decompress inputs: $(find corpus/corpus/zxc_fuzzer_decompress -type f | wc -l)" echo "seekable inputs: $(find corpus/corpus/zxc_fuzzer_seekable -type f | wc -l)" echo "pstream inputs: $(find corpus/corpus/zxc_fuzzer_pstream -type f | wc -l)" - name: Build Coverage Harnesses run: | mkdir -p build-cov CFLAGS="-g -O1 -fprofile-instr-generate -fcoverage-mapping" DEFS="-DZXC_FUNCTION_SUFFIX=_default -DZXC_ONLY_DEFAULT" INCLUDES="-I include -I src/lib/vendors" SOURCES="src/lib/zxc_common.c src/lib/zxc_compress.c src/lib/zxc_decompress.c src/lib/zxc_driver.c src/lib/zxc_dispatch.c src/lib/zxc_huffman.c src/lib/zxc_seekable.c src/lib/zxc_pstream.c" clang $CFLAGS $INCLUDES $DEFS $SOURCES tests/fuzz_roundtrip.c \ -fsanitize=fuzzer -lm -lpthread -o build-cov/fuzz_roundtrip clang $CFLAGS $INCLUDES $DEFS $SOURCES tests/fuzz_decompress.c \ -fsanitize=fuzzer -lm -lpthread -o build-cov/fuzz_decompress clang $CFLAGS $INCLUDES $DEFS $SOURCES tests/fuzz_seekable.c \ -fsanitize=fuzzer -lm -lpthread -o build-cov/fuzz_seekable clang $CFLAGS $INCLUDES $DEFS $SOURCES tests/fuzz_pstream.c \ -fsanitize=fuzzer -lm -lpthread -o build-cov/fuzz_pstream - name: Replay Corpora run: | LLVM_PROFILE_FILE="build-cov/roundtrip.profraw" \ build-cov/fuzz_roundtrip corpus/corpus/zxc_fuzzer_roundtrip/ -runs=0 2>&1 | tail -1 LLVM_PROFILE_FILE="build-cov/decompress.profraw" \ build-cov/fuzz_decompress corpus/corpus/zxc_fuzzer_decompress/ -runs=0 2>&1 | tail -1 if [ -d corpus/corpus/zxc_fuzzer_seekable ]; then LLVM_PROFILE_FILE="build-cov/seekable.profraw" \ build-cov/fuzz_seekable corpus/corpus/zxc_fuzzer_seekable/ -runs=0 2>&1 | tail -1 fi if [ -d corpus/corpus/zxc_fuzzer_pstream ]; then LLVM_PROFILE_FILE="build-cov/pstream.profraw" \ build-cov/fuzz_pstream corpus/corpus/zxc_fuzzer_pstream/ -runs=0 2>&1 | tail -1 fi - name: Generate Coverage Report run: | llvm-profdata merge \ build-cov/roundtrip.profraw build-cov/decompress.profraw \ $([ -f build-cov/seekable.profraw ] && echo build-cov/seekable.profraw) \ $([ -f build-cov/pstream.profraw ] && echo build-cov/pstream.profraw) \ -o build-cov/fuzz.profdata echo "=== COVERAGE SUMMARY ===" llvm-cov report \ build-cov/fuzz_roundtrip -object build-cov/fuzz_decompress \ $([ -f build-cov/fuzz_seekable ] && echo "-object build-cov/fuzz_seekable") \ $([ -f build-cov/fuzz_pstream ] && echo "-object build-cov/fuzz_pstream") \ -instr-profile=build-cov/fuzz.profdata \ --ignore-filename-regex='(vendors/|tests/|fuzz_|zxc_driver)' llvm-cov show \ build-cov/fuzz_roundtrip -object build-cov/fuzz_decompress \ $([ -f build-cov/fuzz_seekable ] && echo "-object build-cov/fuzz_seekable") \ $([ -f build-cov/fuzz_pstream ] && echo "-object build-cov/fuzz_pstream") \ -instr-profile=build-cov/fuzz.profdata \ --ignore-filename-regex='(vendors/|tests/|fuzz_|zxc_driver)' \ --format=html -output-dir=build-cov/coverage_html - name: Upload Coverage Report uses: actions/upload-artifact@v7 with: name: fuzz-coverage-report path: build-cov/coverage_html/ retention-days: 30 zxc-0.11.0/.github/workflows/multiarch.yml000066400000000000000000000300471520102567100205100ustar00rootroot00000000000000name: Multi-Architecture Build on: workflow_dispatch: push: branches: [ main ] pull_request: branches: [ main ] permissions: contents: read concurrency: cancel-in-progress: true group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} jobs: # =========================================================================== # Tier 1: Main architectures # =========================================================================== build-tier1: name: "Tier 1: ${{ matrix.name }}" runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: include: - name: Linux x64 runner: ubuntu-latest os: linux arch_flags: "" - name: Linux x86 (32-bit) runner: ubuntu-latest os: linux install_32bit_x86: true arch_flags: "-m32" - name: Linux ARM64 runner: ubuntu-24.04-arm os: linux arch_flags: "" - name: Linux ARM (32-bit) runner: ubuntu-latest os: linux install_32bit_arm: true cross_compiler_c: arm-linux-gnueabihf-gcc cross_compiler_cxx: arm-linux-gnueabihf-g++ cross_strip: arm-linux-gnueabihf-strip cmake_system_name: "Linux" cmake_system_processor: "arm" - name: Windows x64 runner: windows-latest os: windows cmake_arch: "x64" - name: Windows x86 (32-bit) runner: windows-latest os: windows cmake_arch: "Win32" - name: Windows ARM64 runner: windows-11-arm os: windows cmake_arch: "ARM64" # Note: Windows ARM 32-bit removed - deprecated and unsupported by Windows SDK 10.0.26100+ - name: macOS Intel x64 runner: macos-26-intel os: macos - name: macOS ARM64 runner: macos-26 os: macos steps: - name: Checkout Repository uses: actions/checkout@v6 # Case: Linux x86 (32-bit) - name: Install x86 32-bit libs if: matrix.install_32bit_x86 run: | sudo apt-get update sudo apt-get install -y gcc-multilib g++-multilib # Case: Linux ARM (32-bit) - name: Install ARM 32-bit toolchain if: matrix.install_32bit_arm run: | sudo apt-get update sudo apt-get install -y gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf - name: Configure, Build & Test (Static & Shared) shell: bash run: | # Initialize variables EXTRA_ARGS="-DZXC_NATIVE_ARCH=OFF" # Disable native arch detection (Runtime Dispatch enabled) # Setup Environment for Linux/Mac (Flags & Cross-compilation) if [ "${{ matrix.os }}" != "windows" ]; then # Handle C flags (e.g., -m32) if [ ! -z "${{ matrix.arch_flags }}" ]; then export CFLAGS="${{ matrix.arch_flags }}" export CXXFLAGS="${{ matrix.arch_flags }}" fi # Handle cross-compiler (e.g., ARM 32-bit on Linux) if [ ! -z "${{ matrix.cross_compiler_c }}" ]; then EXTRA_ARGS="$EXTRA_ARGS -DCMAKE_C_COMPILER=${{ matrix.cross_compiler_c }} -DCMAKE_CXX_COMPILER=${{ matrix.cross_compiler_cxx }}" fi # Handle explicit System Name/Processor (crucial for cross-compilation) if [ ! -z "${{ matrix.cmake_system_name }}" ]; then EXTRA_ARGS="$EXTRA_ARGS -DCMAKE_SYSTEM_NAME=${{ matrix.cmake_system_name }}" fi if [ ! -z "${{ matrix.cmake_system_processor }}" ]; then EXTRA_ARGS="$EXTRA_ARGS -DCMAKE_SYSTEM_PROCESSOR=${{ matrix.cmake_system_processor }}" fi # Handle cross-strip tool (e.g., ARM 32-bit on Linux) if [ ! -z "${{ matrix.cross_strip }}" ]; then EXTRA_ARGS="$EXTRA_ARGS -DCMAKE_STRIP=/usr/bin/${{ matrix.cross_strip }}" fi fi # Iterate over Library Types for LIB_TYPE in "STATIC" "SHARED"; do echo ">>> Starting Build: $LIB_TYPE" BUILD_DIR="build_${LIB_TYPE}" SHARED_FLAG="OFF" if [ "$LIB_TYPE" == "SHARED" ]; then SHARED_FLAG="ON"; fi CURRENT_ARGS="$EXTRA_ARGS -DBUILD_SHARED_LIBS=$SHARED_FLAG" # 1. Configure if [ "${{ matrix.os }}" = "windows" ]; then # Visual Studio Generator cmake -S . -B $BUILD_DIR -A ${{ matrix.cmake_arch }} $CURRENT_ARGS else # Ninja/Makefiles cmake -S . -B $BUILD_DIR -DCMAKE_BUILD_TYPE=Release $CURRENT_ARGS fi # 2. Build cmake --build $BUILD_DIR --config Release --parallel # 3. Test (Skip if cross-compiling) if [ -z "${{ matrix.cross_compiler_c }}" ]; then echo "Running Tests for $LIB_TYPE..." ctest --test-dir $BUILD_DIR -C Release --output-on-failure else echo "Skipping tests for $LIB_TYPE (Cross-compilation)" fi echo ">>> Completed Build: $LIB_TYPE" echo "" done # =========================================================================== # Tier 2: Extended Linux architectures (cross-compiled, tested via QEMU) # =========================================================================== build-tier2: name: "Tier 2: ${{ matrix.name }}" runs-on: ubuntu-latest container: debian:trixie strategy: fail-fast: false matrix: include: - name: Linux amd64 cross_pkg: gcc cross_prefix: x86_64-linux-gnu cmake_processor: x86_64 qemu_binary: qemu-x86_64 - name: Linux arm64 cross_pkg: gcc-aarch64-linux-gnu cross_prefix: aarch64-linux-gnu cmake_processor: aarch64 qemu_binary: qemu-aarch64 - name: Linux i386 cross_pkg: gcc-i686-linux-gnu cross_prefix: i686-linux-gnu cmake_processor: i686 qemu_binary: qemu-i386 - name: Linux RISC-V 64 cross_pkg: gcc-riscv64-linux-gnu cross_prefix: riscv64-linux-gnu cmake_processor: riscv64 qemu_binary: qemu-riscv64 - name: Linux s390x cross_pkg: gcc-s390x-linux-gnu cross_prefix: s390x-linux-gnu cmake_processor: s390x qemu_binary: qemu-s390x - name: Linux armhf cross_pkg: gcc-arm-linux-gnueabihf cross_prefix: arm-linux-gnueabihf cmake_processor: arm qemu_binary: qemu-arm - name: Linux ppc64el cross_pkg: gcc-powerpc64le-linux-gnu cross_prefix: powerpc64le-linux-gnu cmake_processor: ppc64le qemu_binary: qemu-ppc64le - name: Linux mipsel cross_pkg: gcc-mipsel-linux-gnu cross_prefix: mipsel-linux-gnu cmake_processor: mipsel qemu_binary: qemu-mipsel - name: Linux powerpc cross_pkg: gcc-powerpc-linux-gnu cross_prefix: powerpc-linux-gnu cmake_processor: powerpc qemu_binary: qemu-ppc - name: Linux ppc64 cross_pkg: gcc-powerpc64-linux-gnu cross_prefix: powerpc64-linux-gnu cmake_processor: ppc64 qemu_binary: qemu-ppc64 - name: Linux sparc64 cross_pkg: gcc-sparc64-linux-gnu cross_prefix: sparc64-linux-gnu cmake_processor: sparc64 qemu_binary: qemu-sparc64 - name: Linux armel cross_pkg: gcc-arm-linux-gnueabi cross_prefix: arm-linux-gnueabi cmake_processor: arm qemu_binary: qemu-arm steps: - name: Checkout Repository uses: actions/checkout@v6 - name: Install cross-compiler run: | apt-get update apt-get install -y cmake build-essential ${{ matrix.cross_pkg }} qemu-user - name: Configure, Build & Test (Static & Shared) shell: bash run: | CROSS_C=${{ matrix.cross_prefix }}-gcc CROSS_CXX=${{ matrix.cross_prefix }}-g++ CROSS_STRIP=/usr/bin/${{ matrix.cross_prefix }}-strip SYSROOT=/usr/${{ matrix.cross_prefix }} for LIB_TYPE in "STATIC" "SHARED"; do echo ">>> Starting Build: $LIB_TYPE" BUILD_DIR="build_${LIB_TYPE}" SHARED_FLAG="OFF" if [ "$LIB_TYPE" == "SHARED" ]; then SHARED_FLAG="ON"; fi cmake -S . -B $BUILD_DIR \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_C_COMPILER=$CROSS_C \ -DCMAKE_CXX_COMPILER=$CROSS_CXX \ -DCMAKE_STRIP=$CROSS_STRIP \ -DCMAKE_SYSTEM_NAME=Linux \ -DCMAKE_SYSTEM_PROCESSOR=${{ matrix.cmake_processor }} \ -DZXC_NATIVE_ARCH=OFF \ -DBUILD_SHARED_LIBS=$SHARED_FLAG cmake --build $BUILD_DIR --config Release --parallel echo "Running Tests for $LIB_TYPE via QEMU..." ${{ matrix.qemu_binary }} -L $SYSROOT ./$BUILD_DIR/zxc_test echo ">>> Completed Build: $LIB_TYPE" echo "" done # =========================================================================== # Tier 3: Experimental Linux architectures (cross-compiled, tested via QEMU) # =========================================================================== build-tier3: name: "Tier 3: ${{ matrix.name }}" runs-on: ubuntu-latest container: debian:trixie strategy: fail-fast: false matrix: include: - name: Linux mips64el cross_pkg: gcc-mips64el-linux-gnuabi64 cross_prefix: mips64el-linux-gnuabi64 cmake_processor: mips64 qemu_binary: qemu-mips64el - name: Linux loong64 cross_pkg: gcc-loongarch64-linux-gnu cross_prefix: loongarch64-linux-gnu cmake_processor: loongarch64 qemu_binary: qemu-loongarch64 - name: Linux alpha cross_pkg: gcc-alpha-linux-gnu cross_prefix: alpha-linux-gnu cmake_processor: alpha qemu_binary: qemu-alpha steps: - name: Checkout Repository uses: actions/checkout@v6 - name: Install cross-compiler run: | apt-get update apt-get install -y cmake build-essential ${{ matrix.cross_pkg }} qemu-user - name: Configure, Build & Test (Static & Shared) shell: bash run: | CROSS_C=${{ matrix.cross_prefix }}-gcc CROSS_CXX=${{ matrix.cross_prefix }}-g++ CROSS_STRIP=/usr/bin/${{ matrix.cross_prefix }}-strip SYSROOT=/usr/${{ matrix.cross_prefix }} for LIB_TYPE in "STATIC" "SHARED"; do echo ">>> Starting Build: $LIB_TYPE" BUILD_DIR="build_${LIB_TYPE}" SHARED_FLAG="OFF" if [ "$LIB_TYPE" == "SHARED" ]; then SHARED_FLAG="ON"; fi cmake -S . -B $BUILD_DIR \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_C_COMPILER=$CROSS_C \ -DCMAKE_CXX_COMPILER=$CROSS_CXX \ -DCMAKE_STRIP=$CROSS_STRIP \ -DCMAKE_SYSTEM_NAME=Linux \ -DCMAKE_SYSTEM_PROCESSOR=${{ matrix.cmake_processor }} \ -DZXC_NATIVE_ARCH=OFF \ -DBUILD_SHARED_LIBS=$SHARED_FLAG cmake --build $BUILD_DIR --config Release --parallel echo "Running Tests for $LIB_TYPE via QEMU..." ${{ matrix.qemu_binary }} -L $SYSROOT ./$BUILD_DIR/zxc_test echo ">>> Completed Build: $LIB_TYPE" echo "" donezxc-0.11.0/.github/workflows/multicomp.yml000066400000000000000000000253321520102567100205320ustar00rootroot00000000000000name: Compiler Compatibility on: workflow_dispatch: push: branches: [ main ] pull_request: branches: [ main ] permissions: contents: read concurrency: cancel-in-progress: true group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} jobs: # =========================================================================== # Clang 12–20 on Ubuntu # =========================================================================== clang: name: "Clang ${{ matrix.version }} (${{ matrix.os }})" runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: # Ubuntu 22.04 ships Clang 13, 14, 15 - Clang 12 is in standard repos - { os: ubuntu-22.04, version: 12, needs_install: true, needs_llvm_apt: false, llvm_distro: "" } - { os: ubuntu-22.04, version: 13, needs_llvm_apt: false, llvm_distro: "" } - { os: ubuntu-22.04, version: 14, needs_llvm_apt: false, llvm_distro: "" } - { os: ubuntu-22.04, version: 15, needs_llvm_apt: false, llvm_distro: "" } # Ubuntu 24.04 ships Clang 16, 17, 18 - 19/20 need LLVM APT - { os: ubuntu-latest, version: 16, needs_llvm_apt: false, llvm_distro: "" } - { os: ubuntu-latest, version: 17, needs_llvm_apt: false, llvm_distro: "" } - { os: ubuntu-latest, version: 18, needs_llvm_apt: false, llvm_distro: "" } - { os: ubuntu-latest, version: 19, needs_llvm_apt: true, llvm_distro: noble } - { os: ubuntu-latest, version: 20, needs_llvm_apt: true, llvm_distro: noble } steps: - name: Checkout Repository uses: actions/checkout@v6 - name: Install Clang ${{ matrix.version }} (standard repos) if: matrix.needs_install && !matrix.needs_llvm_apt run: | sudo apt-get update sudo apt-get install -y clang-${{ matrix.version }} - name: Install Clang ${{ matrix.version }} (LLVM APT) if: matrix.needs_llvm_apt run: | wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc echo "deb http://apt.llvm.org/${{ matrix.llvm_distro }}/ llvm-toolchain-${{ matrix.llvm_distro }}-${{ matrix.version }} main" | sudo tee /etc/apt/sources.list.d/llvm.list sudo apt-get update sudo apt-get install -y clang-${{ matrix.version }} - name: Configure, Build & Test (Static & Shared) shell: bash run: | for LIB_TYPE in "STATIC" "SHARED"; do echo ">>> Starting Build: Clang ${{ matrix.version }} – $LIB_TYPE" BUILD_DIR="build_${LIB_TYPE}" SHARED_FLAG="OFF" if [ "$LIB_TYPE" == "SHARED" ]; then SHARED_FLAG="ON"; fi cmake -S . -B $BUILD_DIR \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_C_COMPILER=clang-${{ matrix.version }} \ -DCMAKE_C_STANDARD=17 \ -DZXC_NATIVE_ARCH=OFF \ -DZXC_ENABLE_LTO=OFF \ -DBUILD_SHARED_LIBS=$SHARED_FLAG cmake --build $BUILD_DIR --config Release --parallel echo "Running Tests for $LIB_TYPE..." ctest --test-dir $BUILD_DIR -C Release --output-on-failure echo ">>> Completed Build: $LIB_TYPE" echo "" done # =========================================================================== # Apple Clang on macOS 14 # =========================================================================== clang-macos: name: "Apple Clang (macOS 14)" runs-on: macos-14 strategy: fail-fast: false steps: - name: Checkout Repository uses: actions/checkout@v6 - name: Configure, Build & Test (Static & Shared) shell: bash run: | for LIB_TYPE in "STATIC" "SHARED"; do echo ">>> Starting Build: Apple Clang – $LIB_TYPE" BUILD_DIR="build_${LIB_TYPE}" SHARED_FLAG="OFF" if [ "$LIB_TYPE" == "SHARED" ]; then SHARED_FLAG="ON"; fi cmake -S . -B $BUILD_DIR \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_C_STANDARD=17 \ -DZXC_NATIVE_ARCH=OFF \ -DBUILD_SHARED_LIBS=$SHARED_FLAG cmake --build $BUILD_DIR --config Release --parallel echo "Running Tests for $LIB_TYPE..." ctest --test-dir $BUILD_DIR -C Release --output-on-failure echo ">>> Completed Build: $LIB_TYPE" echo "" done # =========================================================================== # GCC 9–14 on Ubuntu # =========================================================================== gcc: name: "GCC ${{ matrix.version }} (${{ matrix.os }})" runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: # Ubuntu 22.04 preinstalls GCC 10, 11, 12; GCC 9 lives in universe. - { os: ubuntu-22.04, version: 9, needs_install: true } - { os: ubuntu-22.04, version: 10 } - { os: ubuntu-22.04, version: 11 } - { os: ubuntu-22.04, version: 12 } # Ubuntu 24.04 preinstalls GCC 12, 13, 14. - { os: ubuntu-latest, version: 13 } - { os: ubuntu-latest, version: 14 } steps: - name: Checkout Repository uses: actions/checkout@v6 - name: Install GCC ${{ matrix.version }} if: matrix.needs_install run: | sudo apt-get update sudo apt-get install -y gcc-${{ matrix.version }} g++-${{ matrix.version }} - name: Configure, Build & Test (Static & Shared) shell: bash run: | for LIB_TYPE in "STATIC" "SHARED"; do echo ">>> Starting Build: GCC ${{ matrix.version }} – $LIB_TYPE" BUILD_DIR="build_${LIB_TYPE}" SHARED_FLAG="OFF" if [ "$LIB_TYPE" == "SHARED" ]; then SHARED_FLAG="ON"; fi cmake -S . -B $BUILD_DIR \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_C_COMPILER=gcc-${{ matrix.version }} \ -DCMAKE_CXX_COMPILER=g++-${{ matrix.version }} \ -DCMAKE_C_STANDARD=17 \ -DZXC_NATIVE_ARCH=OFF \ -DZXC_ENABLE_LTO=OFF \ -DBUILD_SHARED_LIBS=$SHARED_FLAG cmake --build $BUILD_DIR --config Release --parallel echo "Running Tests for $LIB_TYPE..." ctest --test-dir $BUILD_DIR -C Release --output-on-failure echo ">>> Completed Build: $LIB_TYPE" echo "" done # =========================================================================== # MinGW on Windows (32-bit & 64-bit) # =========================================================================== mingw: name: "MinGW ${{ matrix.name }}" runs-on: windows-latest strategy: fail-fast: false matrix: include: - name: "x86 32-bit" msystem: MINGW32 pkg_prefix: mingw-w64-i686 - name: "x86_64 64-bit" msystem: MINGW64 pkg_prefix: mingw-w64-x86_64 defaults: run: shell: msys2 {0} steps: - name: Setup MSYS2 uses: msys2/setup-msys2@v2 with: msystem: ${{ matrix.msystem }} update: true install: >- ${{ matrix.pkg_prefix }}-gcc ${{ matrix.pkg_prefix }}-cmake make - name: Checkout Repository uses: actions/checkout@v6 - name: Configure, Build & Test (Static & Shared) run: | for LIB_TYPE in "STATIC" "SHARED"; do echo ">>> Starting Build: MinGW ${{ matrix.name }} – $LIB_TYPE" BUILD_DIR="build_${LIB_TYPE}" SHARED_FLAG="OFF" if [ "$LIB_TYPE" == "SHARED" ]; then SHARED_FLAG="ON"; fi cmake -S . -B $BUILD_DIR \ -G "MSYS Makefiles" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_C_STANDARD=17 \ -DZXC_NATIVE_ARCH=OFF \ -DBUILD_SHARED_LIBS=$SHARED_FLAG cmake --build $BUILD_DIR --config Release --parallel echo "Running Tests for $LIB_TYPE..." ctest --test-dir $BUILD_DIR -C Release --output-on-failure echo ">>> Completed Build: $LIB_TYPE" echo "" done # =========================================================================== # GCC ARM cross-compilation on Ubuntu (32-bit & 64-bit) # =========================================================================== gcc-arm: name: "GCC ARM ${{ matrix.name }}" runs-on: ubuntu-latest strategy: fail-fast: false matrix: include: - name: "32-bit (armhf)" cross_pkg: gcc-arm-linux-gnueabihf cross_prefix: arm-linux-gnueabihf cmake_processor: arm qemu_binary: qemu-arm - name: "64-bit (aarch64)" cross_pkg: gcc-aarch64-linux-gnu cross_prefix: aarch64-linux-gnu cmake_processor: aarch64 qemu_binary: qemu-aarch64 steps: - name: Checkout Repository uses: actions/checkout@v6 - name: Install cross-compiler & QEMU run: | sudo apt-get update sudo apt-get install -y ${{ matrix.cross_pkg }} qemu-user - name: Configure, Build & Test (Static & Shared) shell: bash run: | CROSS_C=${{ matrix.cross_prefix }}-gcc CROSS_CXX=${{ matrix.cross_prefix }}-g++ CROSS_STRIP=/usr/bin/${{ matrix.cross_prefix }}-strip SYSROOT=/usr/${{ matrix.cross_prefix }} for LIB_TYPE in "STATIC" "SHARED"; do echo ">>> Starting Build: GCC ARM ${{ matrix.name }} – $LIB_TYPE" BUILD_DIR="build_${LIB_TYPE}" SHARED_FLAG="OFF" if [ "$LIB_TYPE" == "SHARED" ]; then SHARED_FLAG="ON"; fi cmake -S . -B $BUILD_DIR \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_C_COMPILER=$CROSS_C \ -DCMAKE_CXX_COMPILER=$CROSS_CXX \ -DCMAKE_STRIP=$CROSS_STRIP \ -DCMAKE_SYSTEM_NAME=Linux \ -DCMAKE_SYSTEM_PROCESSOR=${{ matrix.cmake_processor }} \ -DCMAKE_C_STANDARD=17 \ -DZXC_NATIVE_ARCH=OFF \ -DBUILD_SHARED_LIBS=$SHARED_FLAG cmake --build $BUILD_DIR --config Release --parallel echo "Running Tests for $LIB_TYPE via QEMU..." ${{ matrix.qemu_binary }} -L $SYSROOT ./$BUILD_DIR/zxc_test echo ">>> Completed Build: $LIB_TYPE" echo "" done zxc-0.11.0/.github/workflows/quality.yml000066400000000000000000000074151520102567100202130ustar00rootroot00000000000000name: Code Quality on: workflow_dispatch: push: branches: [ main ] pull_request: branches: [ main ] permissions: contents: read concurrency: cancel-in-progress: true group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} jobs: format: name: Code Quality - Clang-Format runs-on: ubuntu-latest steps: - name: Checkout Repository uses: actions/checkout@v6 - name: Install clang-format 22 run: | wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-22 main" | sudo tee /etc/apt/sources.list.d/llvm-22.list sudo apt-get update sudo apt-get install -y clang-format-22 - name: Check Formatting run: make format-check env: CLANG_FORMAT: clang-format-22 cppcheck: name: Code Quality - Cppcheck runs-on: ubuntu-latest steps: - name: Checkout Repository uses: actions/checkout@v6 - name: Install Cppcheck and Cross-Compiler run: | sudo apt-get update sudo apt-get install -y cppcheck gcc-aarch64-linux-gnu libc6-dev-arm64-cross - name: Configure CMake (x86 Compile Database) run: mkdir build_x86 && cd build_x86 && cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON .. - name: Configure CMake (ARM64 Compile Database) run: | mkdir build_arm && cd build_arm cmake -DCMAKE_SYSTEM_NAME=Linux \ -DCMAKE_SYSTEM_PROCESSOR=aarch64 \ -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON .. - name: Run Cppcheck (x86 Variants) run: | cppcheck --project=build_x86/compile_commands.json \ --enable=all \ --suppress=missingIncludeSystem --suppress=*:src/lib/vendors/rapidhash.h \ --check-level=exhaustive \ --inline-suppr \ --error-exitcode=1 \ -j4 - name: Run Cppcheck (ARM64 Variants) run: | cppcheck --project=build_arm/compile_commands.json \ --enable=all \ --suppress=missingIncludeSystem --suppress=*:src/lib/vendors/rapidhash.h \ --check-level=exhaustive \ --inline-suppr \ --error-exitcode=1 \ -j4 quality-checks: name: Code Quality - Advanced Checks runs-on: ubuntu-latest steps: - name: Checkout Repository uses: actions/checkout@v6 - name: Install Analysis Tools run: | sudo apt-get update sudo apt-get install -y clang-tools valgrind - name: Clang Static Analyzer (Scan-Build) run: | mkdir -p build_clang scan-build cmake -S . -B build_clang -DCMAKE_BUILD_TYPE=Debug scan-build --status-bugs cmake --build build_clang - name: Build for Valgrind (Debug) run: | cmake -S . -B build_valgrind -DCMAKE_BUILD_TYPE=Debug -DZXC_NATIVE_ARCH=OFF cmake --build build_valgrind - name: Valgrind Memory Check working-directory: build_valgrind run: | valgrind --leak-check=full \ --show-leak-kinds=all \ --track-origins=yes \ --error-exitcode=1 \ ./zxc_test - name: Unicode Linting shell: bash run: | echo "Scanning for non-ASCII characters in source files..." if grep -rP --include="*.c" --include="*.h" "[^\x00-\x7F]" .; then echo "::error::Non-ASCII characters found in source files." exit 1 else echo "No non-ASCII characters found." fizxc-0.11.0/.github/workflows/security.yml000066400000000000000000000031621520102567100203650ustar00rootroot00000000000000name: Code Security on: workflow_dispatch: push: branches: [ main ] pull_request: branches: [ main ] permissions: contents: read security-events: write jobs: analyze: name: Analyze (${{ matrix.language }}) runs-on: ubuntu-latest timeout-minutes: 10 strategy: fail-fast: false matrix: language: [ 'c-cpp', 'rust', 'python', 'go', 'javascript-typescript' ] steps: - name: Checkout Repository uses: actions/checkout@v6 - name: Initialize CodeQL uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} config-file: ./.github/codeql/codeql-config.yml - name: Build Project run: | if [ "${{ matrix.language }}" = "c-cpp" ]; then make elif [ "${{ matrix.language }}" = "rust" ]; then cd wrappers/rust cargo build --workspace --all-targets elif [ "${{ matrix.language }}" = "python" ]; then cd wrappers/python python3 -m pip install -e . elif [ "${{ matrix.language }}" = "go" ]; then cmake -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DZXC_BUILD_CLI=OFF -DZXC_BUILD_TESTS=OFF cmake --build build --parallel cd wrappers/go CGO_ENABLED=1 CGO_LDFLAGS="-L../../build -lzxc -lpthread" go build ./... elif [ "${{ matrix.language }}" = "javascript-typescript" ]; then cd wrappers/nodejs npm install fi - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4 with: category: "/language:${{ matrix.language }}" zxc-0.11.0/.github/workflows/vendors.yml000066400000000000000000000017571520102567100202060ustar00rootroot00000000000000name: Update Vendors on: workflow_dispatch: schedule: - cron: '0 4 * * 1' jobs: update-rapidhash-header: runs-on: ubuntu-slim permissions: contents: write pull-requests: write steps: - name: Checkout repo uses: actions/checkout@v6 - name: Download latest rapidhash.h run: | curl -o src/lib/vendors/rapidhash.h https://raw.githubusercontent.com/Nicoshev/rapidhash/refs/heads/master/rapidhash.h - name: Create Pull Request uses: peter-evans/create-pull-request@v8 with: token: ${{ secrets.GITHUB_TOKEN }} commit-message: "chore(deps): update rapidhash.h from upstream" title: "Automatic update of rapidhash.h" body: | This PR updates `rapidhash.h` with the latest version from the official repository. Source : [Nicoshev/rapidhash](https://github.com/Nicoshev/rapidhash) branch: auto-update/rapidhash delete-branch: truezxc-0.11.0/.github/workflows/wrapper-go-test.yml000066400000000000000000000061041520102567100215550ustar00rootroot00000000000000name: Test Go Package on: workflow_dispatch: release: types: [ published ] permissions: contents: read jobs: test: name: Test on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: - os: ubuntu-latest - os: ubuntu-24.04-arm - os: macos-26 - os: windows-latest - os: windows-11-arm steps: - name: Checkout Repository uses: actions/checkout@v6 - name: Select Latest Compiler (Linux) if: runner.os == 'Linux' run: | echo "CC=gcc-14" >> $GITHUB_ENV echo "CXX=g++-14" >> $GITHUB_ENV gcc-14 --version - name: Select Latest Xcode (macOS) if: runner.os == 'macOS' run: | sudo xcode-select -s /Applications/Xcode_26.4.app clang --version - name: Install llvm-mingw (Windows ARM64) if: matrix.os == 'windows-11-arm' shell: pwsh run: | $LLVM_MINGW_URL = "https://github.com/mstorsjo/llvm-mingw/releases/download/20260407/llvm-mingw-20260407-ucrt-aarch64.zip" Invoke-WebRequest -Uri $LLVM_MINGW_URL -OutFile llvm-mingw.zip Expand-Archive llvm-mingw.zip -DestinationPath C:\llvm-mingw $MINGW_BIN = "C:\llvm-mingw\llvm-mingw-20260407-ucrt-aarch64\bin" echo "$MINGW_BIN" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append echo "CC=aarch64-w64-mingw32-gcc" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - name: Set up Go uses: actions/setup-go@v6 with: go-version: '1.21' cache: false - name: Build ZXC Core Library (Unix) if: runner.os != 'Windows' run: | cmake -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DZXC_BUILD_CLI=OFF -DZXC_BUILD_TESTS=OFF cmake --build build --parallel - name: Build ZXC Core Library (Windows) if: runner.os == 'Windows' run: | cmake -B build -G "MinGW Makefiles" -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DZXC_BUILD_CLI=OFF -DZXC_BUILD_TESTS=OFF cmake --build build --parallel - name: Run Go Tests (Unix) if: runner.os != 'Windows' working-directory: ./wrappers/go run: CGO_ENABLED=1 CGO_LDFLAGS="-L../../build -lzxc -lpthread" go test -v -count=1 ./... - name: Run Go Tests (Windows) if: runner.os == 'Windows' working-directory: ./wrappers/go env: CGO_ENABLED: "1" CGO_LDFLAGS: "-L../../build -lzxc" run: go test -v -count=1 ./... - name: Run Go Benchmarks (Unix) if: runner.os != 'Windows' working-directory: ./wrappers/go run: CGO_ENABLED=1 CGO_LDFLAGS="-L../../build -lzxc -lpthread" go test -bench=. -benchmem -count=1 ./... - name: Run Go Benchmarks (Windows) if: runner.os == 'Windows' working-directory: ./wrappers/go env: CGO_ENABLED: "1" CGO_LDFLAGS: "-L../../build -lzxc" run: go test -bench=. -benchmem -count=1 ./... zxc-0.11.0/.github/workflows/wrapper-nodejs-publish.yml000066400000000000000000000105271520102567100231250ustar00rootroot00000000000000name: Publish Node.js Package on: workflow_dispatch: release: types: [ published ] permissions: contents: read jobs: build: name: Build on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: - os: ubuntu-latest arch: x86_64 - os: ubuntu-24.04-arm arch: aarch64 - os: macos-26 arch: arm64 - os: macos-26-intel arch: x86_64 - os: windows-latest arch: AMD64 - os: windows-11-arm arch: ARM64 defaults: run: working-directory: ./wrappers/nodejs steps: - name: Checkout Repository uses: actions/checkout@v6 - name: Select Latest Compiler (Linux) if: runner.os == 'Linux' run: | echo "CC=gcc-14" >> $GITHUB_ENV echo "CXX=g++-14" >> $GITHUB_ENV gcc-14 --version - name: Select Latest Xcode (macOS) if: runner.os == 'macOS' run: | sudo xcode-select -s /Applications/Xcode_26.4.app clang --version - name: Set up Node.js uses: actions/setup-node@v6 with: node-version: "22" - name: Install dependencies and build run: npm install - name: Run tests run: npx vitest run --globals - name: Package prebuilt addon run: | mkdir -p prebuilds/${{ matrix.os }}-${{ matrix.arch }} cp build/Release/zxc_nodejs.node prebuilds/${{ matrix.os }}-${{ matrix.arch }}/ - name: Upload prebuilt addon uses: actions/upload-artifact@v7 with: name: prebuilt-${{ matrix.os }}-${{ matrix.arch }} path: wrappers/nodejs/prebuilds/ test: name: Test on ${{ matrix.os }} Node ${{ matrix.node }} needs: [build] runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-latest, ubuntu-24.04-arm, macos-26, macos-26-intel, windows-latest, windows-11-arm] node: ["18", "20", "22"] exclude: # Node.js 18 and 20 don't have official Windows ARM64 builds - os: windows-11-arm node: "18" - os: windows-11-arm node: "20" defaults: run: working-directory: ./wrappers/nodejs steps: - name: Checkout Repository uses: actions/checkout@v6 - name: Select Latest Compiler (Linux) if: runner.os == 'Linux' run: | echo "CC=gcc-14" >> $GITHUB_ENV echo "CXX=g++-14" >> $GITHUB_ENV gcc-14 --version - name: Select Latest Xcode (macOS) if: runner.os == 'macOS' run: | sudo xcode-select -s /Applications/Xcode_26.4.app clang --version - name: Set up Node.js uses: actions/setup-node@v6 with: node-version: ${{ matrix.node }} - name: Install dependencies and build run: npm install - name: Run tests run: npx vitest run --globals publish: name: Publish to npm needs: [build, test] runs-on: ubuntu-slim if: github.event_name == 'release' permissions: id-token: write defaults: run: working-directory: ./wrappers/nodejs steps: - name: Checkout Repository uses: actions/checkout@v6 - name: Set up Node.js uses: actions/setup-node@v6 with: node-version: "22" registry-url: "https://registry.npmjs.org" - name: Verify Version Matches Release Tag run: | RELEASE_VERSION="${{ github.event.release.tag_name }}" # Remove 'v' prefix if present RELEASE_VERSION="${RELEASE_VERSION#v}" PACKAGE_VERSION=$(node -p "require('./package.json').version") if [ "$RELEASE_VERSION" != "$PACKAGE_VERSION" ]; then echo "Version mismatch: Release tag is $RELEASE_VERSION but package.json has $PACKAGE_VERSION" exit 1 fi echo "Version matches: $RELEASE_VERSION" - name: Publish to npm run: npm publish --provenance --access public env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - name: Summary if: success() run: | echo "Successfully published zxc-compress to npm!" echo "Package: https://www.npmjs.com/package/zxc-compress" zxc-0.11.0/.github/workflows/wrapper-python-publish.yml000066400000000000000000000104121520102567100231550ustar00rootroot00000000000000name: Publish Python Package on: workflow_dispatch: release: types: [ published ] permissions: contents: read jobs: build_wheels: name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: - os: ubuntu-latest arch: x86_64 - os: ubuntu-24.04-arm arch: aarch64 - os: macos-26 arch: arm64 - os: macos-26-intel arch: x86_64 - os: windows-latest arch: AMD64 - os: windows-11-arm arch: ARM64 steps: - name: Checkout Repository uses: actions/checkout@v6 with: fetch-depth: 0 - name: Select Latest Compiler (Linux) if: runner.os == 'Linux' run: | echo "CC=gcc-14" >> $GITHUB_ENV echo "CXX=g++-14" >> $GITHUB_ENV gcc-14 --version - name: Select Latest Xcode (macOS) if: runner.os == 'macOS' run: | sudo xcode-select -s /Applications/Xcode_26.4.app clang --version - name: Set up Python uses: actions/setup-python@v6 with: python-version: "3.12" - name: Install build tools run: python -m pip install cibuildwheel==3.3.1 setuptools-scm - name: Get version from git id: version run: echo "version=$(python -m setuptools_scm)" >> $GITHUB_OUTPUT working-directory: ./wrappers/python - name: Build wheels run: python -m cibuildwheel wrappers/python --output-dir wheelhouse env: # Build for Python 3.10-3.13 CIBW_BUILD: "cp310-* cp311-* cp312-* cp313-*" # Skip 32-bit, musllinux, and Python 3.10-3.11 on Windows (missing Development.Module) CIBW_SKIP: "*-win32 *-manylinux_i686 *-musllinux_* cp310-win* cp311-win*" CIBW_ARCHS: "${{ matrix.arch }}" CIBW_ENVIRONMENT: "CMAKE_ARGS='-DZXC_NATIVE_ARCH=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON' SETUPTOOLS_SCM_PRETEND_VERSION=${{ steps.version.outputs.version }}" - name: Upload wheels uses: actions/upload-artifact@v7 with: name: wheels-${{ matrix.os }}-${{ matrix.arch }} path: ./wheelhouse/*.whl build_sdist: name: Build source distribution runs-on: ubuntu-slim steps: - name: Checkout Repository uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up Python uses: actions/setup-python@v6 with: python-version: "3.12" - name: Install build run: python -m pip install build - name: Build sdist run: python -m build --sdist working-directory: ./wrappers/python - name: Upload sdist uses: actions/upload-artifact@v7 with: name: sdist path: ./wrappers/python/dist/*.tar.gz test_wheels: name: Test wheels on ${{ matrix.os }} ${{ matrix.python }} needs: [build_wheels] runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-latest, ubuntu-24.04-arm, macos-26, macos-26-intel, windows-latest, windows-11-arm] python: ["3.12", "3.13"] steps: - name: Checkout Repository uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: python-version: ${{ matrix.python }} - name: Download wheels uses: actions/download-artifact@v8 with: path: wheelhouse pattern: wheels-* merge-multiple: true - name: Install wheel and test dependencies run: | pip install --no-index --find-links wheelhouse zxc-compress pip install pytest - name: Run test suite run: pytest wrappers/python/tests/ -v publish: name: Publish to PyPI needs: [build_wheels, build_sdist, test_wheels] runs-on: ubuntu-latest if: github.event_name == 'release' permissions: id-token: write steps: - name: Download all artifacts uses: actions/download-artifact@v8 with: path: dist merge-multiple: true - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: dist/ zxc-0.11.0/.github/workflows/wrapper-rust-publish.yml000066400000000000000000000060251520102567100226360ustar00rootroot00000000000000name: Publish Rust Crates on: workflow_dispatch: release: types: [ published ] permissions: contents: read jobs: test: name: Test on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: - os: ubuntu-latest arch: x86_64 - os: ubuntu-24.04-arm arch: aarch64 - os: macos-26 arch: x86_64 arm64 - os: windows-latest arch: AMD64 - os: windows-11-arm arch: ARM64 defaults: run: working-directory: ./wrappers/rust steps: - name: Checkout Repository uses: actions/checkout@v6 - name: Select Latest Compiler (Linux) if: runner.os == 'Linux' run: | echo "CC=gcc-14" >> $GITHUB_ENV echo "CXX=g++-14" >> $GITHUB_ENV gcc-14 --version - name: Select Latest Xcode (macOS) if: runner.os == 'macOS' run: | sudo xcode-select -s /Applications/Xcode_26.4.app clang --version - name: Install Rust Toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 with: toolchain: stable - name: Run Tests run: cargo test --workspace publish: name: Publish to crates.io needs: [test] runs-on: ubuntu-slim if: github.event_name == 'release' defaults: run: working-directory: ./wrappers/rust steps: - name: Checkout Repository uses: actions/checkout@v6 - name: Install Rust Toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 with: toolchain: stable - name: Verify Version Matches Release Tag run: | RELEASE_VERSION="${{ github.event.release.tag_name }}" # Remove 'v' prefix if present RELEASE_VERSION="${RELEASE_VERSION#v}" CARGO_VERSION=$(grep '^version' Cargo.toml | head -1 | sed 's/.*"\(.*\)"/\1/') if [ "$RELEASE_VERSION" != "$CARGO_VERSION" ]; then echo "Version mismatch: Release tag is $RELEASE_VERSION but Cargo.toml has $CARGO_VERSION" exit 1 fi echo "Version matches: $RELEASE_VERSION" - name: Publish zxc-compress-sys (FFI bindings) run: | cd zxc-sys cargo publish --token ${{ secrets.CARGO_REGISTRY_TOKEN }} continue-on-error: false # Wait for zxc-compress-sys to be available on crates.io before publishing zxc-compress - name: Wait for zxc-compress-sys to propagate run: sleep 30 - name: Publish zxc-compress (safe wrapper) run: | cd zxc cargo publish --token ${{ secrets.CARGO_REGISTRY_TOKEN }} continue-on-error: false - name: Summary if: success() run: | echo "Successfully published:" echo "- zxc-compress-sys (FFI bindings)" echo "- zxc-compress (safe wrapper)" echo "" echo "Crates are now available on crates.io!" zxc-0.11.0/.github/workflows/wrapper-wasm.yml000066400000000000000000000032331520102567100211420ustar00rootroot00000000000000# ZXC - High-performance lossless compression # # Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. # SPDX-License-Identifier: BSD-3-Clause name: WASM Build on: workflow_dispatch: release: types: [ published ] push: branches: [ main ] paths: - 'src/**' - 'include/**' - 'wrappers/wasm/**' - 'CMakeLists.txt' - '.github/workflows/wrapper-wasm.yml' concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: wasm-build: name: Build & Test WASM runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v6 - name: Setup Emscripten uses: mymindstorm/setup-emsdk@v16 with: version: 3.1.56 - name: Verify Emscripten run: emcc --version - name: Configure run: emcmake cmake -B build-wasm -DCMAKE_BUILD_TYPE=Release - name: Build run: cmake --build build-wasm --parallel - name: Verify outputs run: | ls -lh build-wasm/zxc.js build-wasm/zxc.wasm echo "--- WASM file size ---" wc -c build-wasm/zxc.wasm - name: Setup Node.js uses: actions/setup-node@v6 with: node-version: '22' - name: Run Tests run: BUILD_DIR=build-wasm node wrappers/wasm/test.mjs - name: Upload WASM Artifacts uses: actions/upload-artifact@v7 with: name: zxc-wasm path: | build-wasm/zxc.js build-wasm/zxc.wasm wrappers/wasm/zxc_wasm.js wrappers/wasm/package.json wrappers/wasm/README.md retention-days: 30 zxc-0.11.0/.gitignore000066400000000000000000000020711520102567100143640ustar00rootroot00000000000000# Prerequisites *.d # Object files *.o *.ko *.obj *.elf # Linker output *.ilk *.map *.exp # Precompiled Headers *.gch *.pch # Libraries *.lib *.a *.la *.lo # Shared objects (inc. Windows DLLs) *.dll *.so *.so.* *.dylib # Executables *.exe *.out *.app *.i*86 *.x86_64 *.hex # Debug files *.dSYM/ *.su *.idb *.pdb # debug information files *.dwo CMakeLists.txt.user CMakeCache.txt CMakeFiles CMakeScripts Testing Makefile !/Makefile cmake_install.cmake install_manifest.txt compile_commands.json CTestTestfile.cmake _deps CMakeUserPresets.json # CLion # JetBrains specific template is maintained in a separate JetBrains.gitignore that can # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #cmake-build-* build/* # Doxygen docs/html/ docs/latex/ docs/rtf/ docs/man/*.1 docs/xml/ *.doxygen_sqlite3 doxyfile.inc searchdata.xml # Coverage build-cov/ zxc-0.11.0/.snyk000066400000000000000000000001051520102567100133550ustar00rootroot00000000000000# Snyk (https://snyk.io) policy file exclude: code: - tests/** zxc-0.11.0/CMakeLists.txt000066400000000000000000000606161520102567100151450ustar00rootroot00000000000000# ZXC - High-performance lossless compression # # Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. # SPDX-License-Identifier: BSD-3-Clause cmake_minimum_required(VERSION 3.14) # ============================================================================= # Version extraction from header # ============================================================================= file(READ "include/zxc_constants.h" version_header) string(REGEX MATCH "#define ZXC_VERSION_MAJOR ([0-9]+)" _ "${version_header}") set(MAJOR_VER ${CMAKE_MATCH_1}) string(REGEX MATCH "#define ZXC_VERSION_MINOR ([0-9]+)" _ "${version_header}") set(MINOR_VER ${CMAKE_MATCH_1}) string(REGEX MATCH "#define ZXC_VERSION_PATCH ([0-9]+)" _ "${version_header}") set(PATCH_VER ${CMAKE_MATCH_1}) project(zxc VERSION ${MAJOR_VER}.${MINOR_VER}.${PATCH_VER} LANGUAGES C DESCRIPTION "High-performance asymmetric lossless compression library" ) # ============================================================================= # Build Options # ============================================================================= option(BUILD_SHARED_LIBS "Build shared libraries instead of static" OFF) option(ZXC_NATIVE_ARCH "Enable -march=native for maximum performance" ON) option(ZXC_ENABLE_LTO "Enable Interprocedural Optimization (LTO)" ON) set(ZXC_PGO_MODE "OFF" CACHE STRING "Profile-Guided Optimization mode (OFF/GENERATE/USE)") set_property(CACHE ZXC_PGO_MODE PROPERTY STRINGS OFF GENERATE USE) option(ZXC_BUILD_CLI "Build the command-line interface" ON) option(ZXC_BUILD_TESTS "Build unit tests" ON) option(ZXC_ENABLE_COVERAGE "Enable code coverage generation" OFF) option(ZXC_DISABLE_SIMD "Disable explicit SIMD intrinsics (no AVX/NEON code paths)" OFF) # ============================================================================= # Emscripten / WebAssembly overrides # ============================================================================= if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") message(STATUS "Emscripten detected - configuring for WebAssembly.") set(ZXC_DISABLE_SIMD ON CACHE BOOL "" FORCE) set(ZXC_BUILD_CLI OFF CACHE BOOL "" FORCE) set(ZXC_BUILD_TESTS OFF CACHE BOOL "" FORCE) set(ZXC_NATIVE_ARCH OFF CACHE BOOL "" FORCE) set(ZXC_ENABLE_LTO OFF CACHE BOOL "" FORCE) set(ZXC_PGO_MODE "OFF" CACHE STRING "" FORCE) set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) endif() if(ZXC_DISABLE_SIMD) add_compile_definitions(ZXC_DISABLE_SIMD) endif() # ============================================================================= # C Standard # ============================================================================= set(CMAKE_C_STANDARD 17) set(CMAKE_C_STANDARD_REQUIRED ON) set(CMAKE_C_EXTENSIONS OFF) # Enable _GNU_SOURCE for ftello/fseeko on Linux # Enable 64-bit off_t on 32-bit Linux to prevent fseeko/ftello/pread truncation if(UNIX AND NOT APPLE) add_compile_definitions(_GNU_SOURCE _FILE_OFFSET_BITS=64 _LARGEFILE_SOURCE) elseif(APPLE) add_compile_definitions(_GNU_SOURCE) endif() # Check for LTO support if(ZXC_ENABLE_LTO AND NOT ZXC_ENABLE_COVERAGE) include(CheckIPOSupported) check_ipo_supported(RESULT result OUTPUT output) if(result) message(STATUS "LTO/IPO is supported and enabled.") else() message(WARNING "LTO/IPO is not supported: ${output}") set(ZXC_ENABLE_LTO OFF) endif() elseif(ZXC_ENABLE_COVERAGE) message(STATUS "Code coverage enabled: Disabling LTO and PGO.") set(ZXC_ENABLE_LTO OFF) set(ZXC_PGO_MODE "OFF") endif() # ============================================================================= # Rapidhash: system-installed (e.g. vcpkg) or vendored fallback # ============================================================================= find_path(RAPIDHASH_INCLUDE_DIR rapidhash.h) if(NOT RAPIDHASH_INCLUDE_DIR) set(RAPIDHASH_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src/lib/vendors") message(STATUS "Using vendored rapidhash from ${RAPIDHASH_INCLUDE_DIR}") else() message(STATUS "Found system rapidhash in ${RAPIDHASH_INCLUDE_DIR}") endif() # ============================================================================= # Core Library & Runtime Dispatch # ============================================================================= # --- PGO flag selection (Clang vs GCC) --- set(ZXC_PGO_DIR "${CMAKE_BINARY_DIR}/pgo") set(ZXC_PGO_GEN_CFLAGS "") set(ZXC_PGO_GEN_LDFLAGS "") set(ZXC_PGO_USE_CFLAGS "") set(ZXC_PGO_USE_LDFLAGS "") if(NOT MSVC AND NOT ZXC_PGO_MODE STREQUAL "OFF") if(CMAKE_C_COMPILER_ID MATCHES "Clang") # Clang: instrumentation-based PGO set(ZXC_PGO_PROFDATA "${ZXC_PGO_DIR}/default.profdata") set(ZXC_PGO_GEN_CFLAGS -fprofile-instr-generate=${ZXC_PGO_DIR}/default_%m.profraw) set(ZXC_PGO_GEN_LDFLAGS -fprofile-instr-generate) set(ZXC_PGO_USE_CFLAGS -fprofile-instr-use=${ZXC_PGO_PROFDATA}) set(ZXC_PGO_USE_LDFLAGS -fprofile-instr-use=${ZXC_PGO_PROFDATA}) else() # GCC: directory-based PGO set(ZXC_PGO_GEN_CFLAGS -fprofile-generate=${ZXC_PGO_DIR}) set(ZXC_PGO_GEN_LDFLAGS -fprofile-generate=${ZXC_PGO_DIR}) set(ZXC_PGO_USE_CFLAGS -fprofile-use=${ZXC_PGO_DIR} -fprofile-correction) set(ZXC_PGO_USE_LDFLAGS -fprofile-use=${ZXC_PGO_DIR}) endif() endif() # Helper: apply PGO flags to a target macro(zxc_apply_pgo target) if(ZXC_PGO_MODE STREQUAL "GENERATE") target_compile_options(${target} PRIVATE ${ZXC_PGO_GEN_CFLAGS}) target_link_options(${target} PRIVATE ${ZXC_PGO_GEN_LDFLAGS}) elseif(ZXC_PGO_MODE STREQUAL "USE") if(EXISTS "${ZXC_PGO_DIR}") target_compile_options(${target} PRIVATE ${ZXC_PGO_USE_CFLAGS}) target_link_options(${target} PRIVATE ${ZXC_PGO_USE_LDFLAGS}) endif() endif() endmacro() # Function Multi-Versioning Helper # Compiles compress/decompress/huffman with specific flags and suffix so the # runtime dispatcher can route to a BMI2/AVX2/AVX512/NEON-aware Huffman codec # in addition to the LZ77 stages. macro(zxc_add_variant suffix flags) foreach(_src compress decompress huffman) add_library(zxc_${_src}${suffix} OBJECT src/lib/zxc_${_src}.c) target_compile_options(zxc_${_src}${suffix} PRIVATE ${flags}) target_compile_definitions(zxc_${_src}${suffix} PRIVATE ZXC_FUNCTION_SUFFIX=${suffix}) # For static builds, define ZXC_STATIC_DEFINE if(NOT BUILD_SHARED_LIBS) target_compile_definitions(zxc_${_src}${suffix} PRIVATE ZXC_STATIC_DEFINE) else() # Mark as part of the DLL being built (avoids dllimport on internal symbols) target_compile_definitions(zxc_${_src}${suffix} PRIVATE zxc_lib_EXPORTS) set_target_properties(zxc_${_src}${suffix} PROPERTIES POSITION_INDEPENDENT_CODE ON) # Hide variant symbols from shared library public ABI if(NOT MSVC) target_compile_options(zxc_${_src}${suffix} PRIVATE -fvisibility=hidden) endif() endif() # Inherit include directories target_include_directories(zxc_${_src}${suffix} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/lib ${RAPIDHASH_INCLUDE_DIR} PUBLIC $) # Propagate PGO flags to variant objects zxc_apply_pgo(zxc_${_src}${suffix}) list(APPEND ZXC_VARIANT_OBJECTS $) endforeach() endmacro() set(ZXC_VARIANT_OBJECTS "") # --- 1. Default Variant (Scalar/Baseline) --- zxc_add_variant(_default "") # --- 2. Architecture Specific Variants (skipped in no-intrinsics mode) --- if(ZXC_DISABLE_SIMD) message(STATUS "ZXC_DISABLE_SIMD: Skipping SIMD variants (no explicit AVX/NEON code paths).") else() if(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64|x86_64|AMD64") message(STATUS "Building x86_64 AVX2 and AVX512 variants...") if(MSVC) # AVX2 for MSVC (Enables AVX2/BMI1/BMI2 sets) zxc_add_variant(_avx2 "/arch:AVX2;/D__BMI__;/D__BMI2__;/D__LZCNT__") # AVX512 for MSVC (VS2019 16.10+ supports /arch:AVX512) zxc_add_variant(_avx512 "/arch:AVX512;/D__BMI__;/D__BMI2__;/D__LZCNT__") else() # AVX2 for GCC/Clang zxc_add_variant(_avx2 "-mavx2;-mfma;-mbmi;-mbmi2;-mlzcnt") # AVX512 for GCC/Clang zxc_add_variant(_avx512 "-mavx512f;-mavx512bw;-mbmi;-mbmi2;-mlzcnt") endif() elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64") message(STATUS "Building AArch64 NEON variant...") if(MSVC) # MSVC for ARM64 implies NEON support by default. zxc_add_variant(_neon "") else() # NEON is usually default on AArch64, but we add a specific variant for structure zxc_add_variant(_neon "-march=armv8-a+simd") endif() elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^arm") message(STATUS "Building ARM NEON variant...") zxc_add_variant(_neon "-march=armv7-a;-mfpu=neon") endif() endif() # Sources: exclude zxc_driver.c for Emscripten (requires pthreads + FILE* I/O). # zxc_huffman.c lives in the per-variant build (see zxc_add_variant). if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") set(ZXC_CORE_SOURCES src/lib/zxc_common.c src/lib/zxc_dispatch.c src/lib/zxc_pstream.c src/lib/zxc_seekable.c ) else() set(ZXC_CORE_SOURCES src/lib/zxc_common.c src/lib/zxc_driver.c src/lib/zxc_dispatch.c src/lib/zxc_pstream.c src/lib/zxc_seekable.c ) endif() add_library(zxc_lib ${ZXC_CORE_SOURCES} ${ZXC_VARIANT_OBJECTS} ) # ============================================================================= # ABI Versioning (Debian/Linux shared libraries) # ============================================================================= # Increment this number ONLY when breaking the ABI. set(ZXC_SOVERSION 3) # Set library output name and version set_target_properties(zxc_lib PROPERTIES OUTPUT_NAME zxc VERSION ${PROJECT_VERSION} SOVERSION ${ZXC_SOVERSION} ) # Target-based include directories for the main lib target_include_directories(zxc_lib PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/lib ${RAPIDHASH_INCLUDE_DIR} ) # Symbol visibility for shared libraries if(BUILD_SHARED_LIBS) # Set visibility for GCC/Clang if(NOT MSVC) target_compile_options(zxc_lib PRIVATE -fvisibility=hidden) set_target_properties(zxc_lib PROPERTIES C_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN YES ) endif() else() # For static libraries, define ZXC_STATIC_DEFINE to avoid dllimport/dllexport target_compile_definitions(zxc_lib PUBLIC ZXC_STATIC_DEFINE) endif() # ============================================================================= # Compiler-specific options (using generator expressions) # ============================================================================= if(NOT MSVC) target_compile_options(zxc_lib PRIVATE $<$>:-O3> -Wall -Wextra -fomit-frame-pointer -fstrict-aliasing # Native Arch $<$:-march=native> # Dead code elimination -ffunction-sections -fdata-sections ) # Profile-Guided Optimization zxc_apply_pgo(zxc_lib) if(ZXC_PGO_MODE STREQUAL "GENERATE") message(STATUS "PGO: Generating profile data to ${ZXC_PGO_DIR}") elseif(ZXC_PGO_MODE STREQUAL "USE") if(NOT EXISTS "${ZXC_PGO_DIR}") message(FATAL_ERROR "PGO: Profile data not found at ${ZXC_PGO_DIR}. Run with ZXC_PGO_MODE=GENERATE first.") endif() message(STATUS "PGO: Using profile data from ${ZXC_PGO_DIR}") endif() # Linker options for Dead Code Stripping if(APPLE) target_link_options(zxc_lib PRIVATE -Wl,-dead_strip) else() target_link_options(zxc_lib PRIVATE -Wl,--gc-sections) endif() else() target_compile_options(zxc_lib PRIVATE $<$:/O2> /W3) target_compile_definitions(zxc_lib PRIVATE _CRT_SECURE_NO_WARNINGS) endif() target_compile_definitions(zxc_lib PRIVATE $<$>:_GNU_SOURCE> ) # Coverage flags if(ZXC_ENABLE_COVERAGE) if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") target_compile_options(zxc_lib PRIVATE --coverage -fprofile-update=atomic) target_link_options(zxc_lib PRIVATE --coverage) endif() endif() # Enable LTO cleanly if(ZXC_ENABLE_LTO) set_property(TARGET zxc_lib PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE) if(NOT MSVC) target_compile_options(zxc_lib PRIVATE -flto) target_link_options(zxc_lib PRIVATE -flto) endif() endif() # Threading support (not available in WASM single-threaded builds) if(NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten") find_package(Threads REQUIRED) target_link_libraries(zxc_lib PRIVATE Threads::Threads) endif() # ============================================================================= # CLI Executable # ============================================================================= if(ZXC_BUILD_CLI) add_executable(zxc src/cli/main.c) target_link_libraries(zxc PRIVATE zxc_lib) target_include_directories(zxc PRIVATE ${RAPIDHASH_INCLUDE_DIR}) # Math library on Unix if(UNIX) target_link_libraries(zxc PRIVATE m) endif() # Propagate compile options and definitions target_compile_options(zxc PRIVATE $<$>,$>:-march=native> ) target_compile_definitions(zxc PRIVATE $<$:_CRT_SECURE_NO_WARNINGS> $<$>:_GNU_SOURCE> ) # Coverage flags for CLI if(ZXC_ENABLE_COVERAGE) if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") target_compile_options(zxc PRIVATE --coverage) target_link_options(zxc PRIVATE --coverage) endif() endif() # Enable LTO cleanly if(ZXC_ENABLE_LTO) set_property(TARGET zxc PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE) if(NOT MSVC) target_compile_options(zxc PRIVATE -flto) target_link_options(zxc PRIVATE -flto) endif() endif() # Profile-Guided Optimization for CLI zxc_apply_pgo(zxc) # Linker options for Dead Code Stripping if(NOT MSVC) if(APPLE) target_link_options(zxc PRIVATE -Wl,-dead_strip) else() target_link_options(zxc PRIVATE -Wl,--gc-sections) endif() endif() # Strip symbols in Release mode for smaller binary if(NOT MSVC AND CMAKE_BUILD_TYPE STREQUAL "Release") # Set default strip command if not already set (e.g., for cross-compilation) if(NOT CMAKE_STRIP) set(CMAKE_STRIP strip) endif() add_custom_command(TARGET zxc POST_BUILD COMMAND ${CMAKE_STRIP} $ COMMENT "Stripping symbols from zxc" ) endif() endif() # ============================================================================= # Tests # ============================================================================= if(ZXC_BUILD_TESTS) enable_testing() add_executable(zxc_test tests/test_main.c tests/test_common.c tests/test_buffer_api.c tests/test_block_api.c tests/test_context_api.c tests/test_pstream_api.c tests/test_stream_api.c tests/test_seekable.c tests/test_seekable_mt.c tests/test_format.c tests/test_misc.c ) # When building shared libraries, create a static version for tests # This allows tests to access internal functions for unit testing if(BUILD_SHARED_LIBS) # Create a static library specifically for tests. # zxc_huffman.c lives in the per-variant build (see zxc_add_variant) # and is already pulled in via ${ZXC_VARIANT_OBJECTS} below. add_library(zxc_lib_static STATIC src/lib/zxc_common.c src/lib/zxc_driver.c src/lib/zxc_dispatch.c src/lib/zxc_pstream.c src/lib/zxc_seekable.c ${ZXC_VARIANT_OBJECTS} ) # Copy all properties from the shared library target_include_directories(zxc_lib_static PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/lib ${RAPIDHASH_INCLUDE_DIR} ) # Apply same compiler settings as main library target_compile_options(zxc_lib_static PRIVATE $ ) target_compile_definitions(zxc_lib_static PUBLIC ZXC_STATIC_DEFINE) target_link_libraries(zxc_lib_static PRIVATE Threads::Threads) # Link tests against static library target_link_libraries(zxc_test PRIVATE zxc_lib_static) else() # For static builds, use the main library target_link_libraries(zxc_test PRIVATE zxc_lib) endif() # Propagate compile options target_compile_options(zxc_test PRIVATE $<$>,$>:-march=native> ) # Coverage flags for Tests if(ZXC_ENABLE_COVERAGE) if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") target_link_options(zxc_test PRIVATE --coverage) endif() endif() # Profile-Guided Optimization for tests zxc_apply_pgo(zxc_test) target_include_directories(zxc_test PRIVATE src/lib ${RAPIDHASH_INCLUDE_DIR}) file(STRINGS tests/test_main.c ZXC_TEST_CASE_LINES REGEX "TEST_CASE\\(") set(ZXC_TEST_NAMES "") foreach(_line IN LISTS ZXC_TEST_CASE_LINES) if(_line MATCHES "TEST_CASE\\(([A-Za-z0-9_]+)\\)") list(APPEND ZXC_TEST_NAMES ${CMAKE_MATCH_1}) endif() endforeach() foreach(_name IN LISTS ZXC_TEST_NAMES) add_test(NAME ${_name} COMMAND zxc_test --exact ${_name}) endforeach() endif() # ============================================================================= # Installation # ============================================================================= include(GNUInstallDirs) configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/libzxc.pc.in ${CMAKE_CURRENT_BINARY_DIR}/libzxc.pc @ONLY ) install(TARGETS zxc_lib EXPORT zxc-targets ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ) install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} FILES_MATCHING PATTERN "*.h" ) if(ZXC_BUILD_CLI) install(TARGETS zxc RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) endif() install(FILES ${CMAKE_CURRENT_BINARY_DIR}/libzxc.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig ) # CMake package config files for find_package(zxc) include(CMakePackageConfigHelpers) install(EXPORT zxc-targets FILE zxc-targets.cmake NAMESPACE zxc:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/zxc ) configure_package_config_file( ${CMAKE_CURRENT_SOURCE_DIR}/zxcConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/zxcConfig.cmake INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/zxc ) write_basic_package_version_file( ${CMAKE_CURRENT_BINARY_DIR}/zxcConfigVersion.cmake VERSION ${PROJECT_VERSION} COMPATIBILITY SameMajorVersion ) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zxcConfig.cmake ${CMAKE_CURRENT_BINARY_DIR}/zxcConfigVersion.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/zxc ) # ============================================================================= # Code Formatting (clang-format) # ============================================================================= # Allow override via environment variable (e.g., CLANG_FORMAT=clang-format-22) if(DEFINED ENV{CLANG_FORMAT}) set(CLANG_FORMAT "$ENV{CLANG_FORMAT}") else() find_program(CLANG_FORMAT clang-format) endif() if(CLANG_FORMAT) file(GLOB_RECURSE ZXC_FORMAT_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/include/*.h" "${CMAKE_CURRENT_SOURCE_DIR}/src/lib/*.c" "${CMAKE_CURRENT_SOURCE_DIR}/src/lib/*.h" ) # Exclude vendored third-party code list(FILTER ZXC_FORMAT_SOURCES EXCLUDE REGEX ".*/vendors/.*") add_custom_target(format COMMAND ${CLANG_FORMAT} --style=file -i ${ZXC_FORMAT_SOURCES} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMENT "Formatting ${CMAKE_CURRENT_SOURCE_DIR}/include and src/lib with clang-format" VERBATIM ) add_custom_target(format-check COMMAND ${CLANG_FORMAT} --style=file --dry-run --Werror ${ZXC_FORMAT_SOURCES} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMENT "Checking formatting of include/ and src/lib/" VERBATIM ) else() message(STATUS "clang-format not found - format/format-check targets disabled") endif() # ============================================================================= # Summary # ============================================================================= message(STATUS "") message(STATUS "ZXC Configuration Summary:") message(STATUS " Version: ${PROJECT_VERSION}") if(BUILD_SHARED_LIBS) message(STATUS " Library Type: Shared") else() message(STATUS " Library Type: Static") endif() message(STATUS " Native Arch: ${ZXC_NATIVE_ARCH}") message(STATUS " Disable SIMD: ${ZXC_DISABLE_SIMD}") message(STATUS " LTO Enabled: ${ZXC_ENABLE_LTO}") message(STATUS " PGO Mode: ${ZXC_PGO_MODE}") message(STATUS " Build CLI: ${ZXC_BUILD_CLI}") message(STATUS " Build Tests: ${ZXC_BUILD_TESTS}") message(STATUS "") # ============================================================================= # WebAssembly (Emscripten) Target # ============================================================================= if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") # Exported C functions (with leading underscore for Emscripten convention) set(ZXC_WASM_EXPORTS "_zxc_compress" "_zxc_decompress" "_zxc_compress_bound" "_zxc_compress_block_bound" "_zxc_get_decompressed_size" "_zxc_create_cctx" "_zxc_free_cctx" "_zxc_compress_cctx" "_zxc_create_dctx" "_zxc_free_dctx" "_zxc_decompress_dctx" # Push streaming API "_zxc_cstream_create" "_zxc_cstream_free" "_zxc_cstream_compress" "_zxc_cstream_end" "_zxc_cstream_in_size" "_zxc_cstream_out_size" "_zxc_dstream_create" "_zxc_dstream_free" "_zxc_dstream_decompress" "_zxc_dstream_finished" "_zxc_dstream_in_size" "_zxc_dstream_out_size" "_zxc_min_level" "_zxc_max_level" "_zxc_default_level" "_zxc_version_string" "_zxc_error_name" "_malloc" "_free" ) # Join list with commas for Emscripten linker flag string(JOIN "," ZXC_WASM_EXPORTS_STR ${ZXC_WASM_EXPORTS}) add_executable(zxc_wasm wrappers/wasm/wasm_entry.c) target_link_libraries(zxc_wasm PRIVATE zxc_lib) set_target_properties(zxc_wasm PROPERTIES OUTPUT_NAME "zxc" SUFFIX ".js" ) target_link_options(zxc_wasm PRIVATE "-sEXPORTED_FUNCTIONS=[${ZXC_WASM_EXPORTS_STR}]" "-sEXPORTED_RUNTIME_METHODS=[ccall,cwrap,getValue,setValue,UTF8ToString,HEAPU8,HEAP32,HEAPU32]" "-sMODULARIZE=1" "-sEXPORT_NAME=ZXCModule" "-sALLOW_MEMORY_GROWTH=1" "-sINITIAL_MEMORY=2097152" "-sSTACK_SIZE=131072" "-sENVIRONMENT=web,node" "-sNO_FILESYSTEM=1" "-sSTRICT=1" ) target_compile_options(zxc_wasm PRIVATE -O3) message(STATUS " WASM Target: zxc.js + zxc.wasm") endif() # ============================================================================= # Documentation (Doxygen) # ============================================================================= find_package(Doxygen) if(DOXYGEN_FOUND) # Generate the Doxyfile with the current project version configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY) # Add a 'doc' target to generate documentation (e.g. 'make doc' or 'cmake --build . --target doc') add_custom_target(doc COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMENT "Generating API documentation with Doxygen" VERBATIM ) endif() zxc-0.11.0/Doxyfile.in000066400000000000000000000251371520102567100145170ustar00rootroot00000000000000# Doxyfile 1.9.1 #--------------------------------------------------------------------------- # Project related configuration options #--------------------------------------------------------------------------- PROJECT_NAME = "@PROJECT_NAME@" PROJECT_NUMBER = "@PROJECT_VERSION@" PROJECT_BRIEF = "@PROJECT_DESCRIPTION@" OUTPUT_DIRECTORY = docs CREATE_SUBDIRS = NO ALLOW_UNICODE_NAMES = NO OUTPUT_LANGUAGE = English BRIEF_MEMBER_DESC = YES REPEAT_BRIEF = YES ABBREVIATE_BRIEF = ALWAYS_DETAILED_SEC = NO INLINE_INHERITED_MEMB = NO FULL_PATH_NAMES = YES STRIP_FROM_PATH = STRIP_FROM_INC_PATH = SHORT_NAMES = NO JAVADOC_AUTOBRIEF = YES JAVADOC_BANNER = NO QT_AUTOBRIEF = NO MULTILINE_CPP_IS_BRIEF = NO PYTHON_DOCSTRING = YES INHERIT_DOCS = YES SEPARATE_MEMBER_PAGES = NO TAB_SIZE = 4 ALIASES = OPTIMIZE_OUTPUT_FOR_C = YES OPTIMIZE_OUTPUT_JAVA = NO OPTIMIZE_FOR_FORTRAN = NO OPTIMIZE_OUTPUT_VHDL = NO OPTIMIZE_OUTPUT_SLICE = NO EXTENSION_MAPPING = MARKDOWN_SUPPORT = YES TOC_INCLUDE_HEADINGS = 5 AUTOLINK_SUPPORT = YES BUILTIN_STL_SUPPORT = NO CPP_CLI_SUPPORT = NO SIP_SUPPORT = NO IDL_PROPERTY_SUPPORT = YES DISTRIBUTE_GROUP_DOC = NO GROUP_NESTED_COMPOUNDS = NO SUBGROUPING = YES INLINE_GROUPED_CLASSES = NO INLINE_SIMPLE_STRUCTS = NO TYPEDEF_HIDES_STRUCT = NO LOOKUP_CACHE_SIZE = 0 NUM_PROC_THREADS = 1 #--------------------------------------------------------------------------- # Build related configuration options #--------------------------------------------------------------------------- EXTRACT_ALL = NO EXTRACT_PRIVATE = NO EXTRACT_PRIV_VIRTUAL = NO EXTRACT_PACKAGE = NO EXTRACT_STATIC = YES EXTRACT_LOCAL_CLASSES = NO EXTRACT_LOCAL_METHODS = NO EXTRACT_ANON_NSPACES = NO RESOLVE_UNNAMED_PARAMS = YES HIDE_UNDOC_MEMBERS = NO HIDE_UNDOC_CLASSES = NO HIDE_FRIEND_COMPOUNDS = NO HIDE_IN_BODY_DOCS = NO INTERNAL_DOCS = NO CASE_SENSE_NAMES = YES HIDE_SCOPE_NAMES = NO HIDE_COMPOUND_REFERENCE= NO SHOW_HEADERFILE = YES SHOW_INCLUDE_FILES = YES SHOW_GROUPED_MEMB_INC = NO FORCE_LOCAL_INCLUDES = NO INLINE_INFO = YES SORT_MEMBER_DOCS = YES SORT_BRIEF_DOCS = NO SORT_MEMBERS_CTORS_1ST = NO SORT_GROUP_NAMES = NO SORT_BY_SCOPE_NAME = NO STRICT_PROTO_MATCHING = NO GENERATE_TODOLIST = YES GENERATE_TESTLIST = YES GENERATE_BUGLIST = YES GENERATE_DEPRECATEDLIST= YES ENABLED_SECTIONS = MAX_INITIALIZER_LINES = 30 SHOW_USED_FILES = YES SHOW_FILES = YES SHOW_NAMESPACES = YES FILE_VERSION_FILTER = LAYOUT_FILE = CITE_BIB_FILES = #--------------------------------------------------------------------------- # Configuration options related to warning and progress messages #--------------------------------------------------------------------------- QUIET = NO WARNINGS = YES WARN_IF_UNDOCUMENTED = NO WARN_IF_DOC_ERROR = YES WARN_IF_INCOMPLETE_DOC = YES WARN_NO_PARAMDOC = YES WARN_AS_ERROR = NO WARN_FORMAT = "$file:$line: $text" WARN_LINE_FORMAT = "at line $line of file $file" WARN_LOGFILE = #--------------------------------------------------------------------------- # Configuration options related to the input files #--------------------------------------------------------------------------- INPUT = include \ src/lib INPUT_ENCODING = UTF-8 INPUT_FILE_ENCODING = FILE_PATTERNS = *.h \ *.c RECURSIVE = YES EXCLUDE = EXCLUDE_SYMLINKS = NO EXCLUDE_PATTERNS = */test/* \ */tests/* \ */build/* \ */target/* \ */src/lib/*.c EXCLUDE_SYMBOLS = EXAMPLE_PATH = EXAMPLE_PATTERNS = * EXAMPLE_RECURSIVE = NO IMAGE_PATH = INPUT_FILTER = FILTER_PATTERNS = FILTER_SOURCE_FILES = NO FILTER_SOURCE_PATTERNS = USE_MDFILE_AS_MAINPAGE = README.md #--------------------------------------------------------------------------- # Configuration options related to source browsing #--------------------------------------------------------------------------- SOURCE_BROWSER = YES INLINE_SOURCES = NO STRIP_CODE_COMMENTS = YES REFERENCED_BY_RELATION = YES REFERENCES_RELATION = YES REFERENCES_LINK_SOURCE = YES SOURCE_TOOLTIPS = YES USE_HTAGS = NO VERBATIM_HEADERS = YES CLANG_ASSISTED_PARSING = NO CLANG_ADD_INC_PATHS = YES CLANG_OPTIONS = CLANG_DATABASE_PATH = #--------------------------------------------------------------------------- # Configuration options related to the alphabetical class index #--------------------------------------------------------------------------- ALPHABETICAL_INDEX = YES IGNORE_PREFIX = #--------------------------------------------------------------------------- # Configuration options related to the HTML output #--------------------------------------------------------------------------- GENERATE_HTML = YES HTML_OUTPUT = html HTML_FILE_EXTENSION = .html HTML_HEADER = HTML_FOOTER = HTML_STYLESHEET = HTML_EXTRA_STYLESHEET = HTML_EXTRA_FILES = HTML_COLORSTYLE = AUTO_LIGHT HTML_COLORSTYLE_HUE = 220 HTML_COLORSTYLE_SAT = 100 HTML_COLORSTYLE_GAMMA = 80 HTML_TIMESTAMP = NO HTML_DYNAMIC_MENUS = YES HTML_DYNAMIC_SECTIONS = NO HTML_INDEX_NUM_ENTRIES = 100 GENERATE_DOCSET = NO GENERATE_HTMLHELP = NO GENERATE_QHP = NO GENERATE_ECLIPSEHELP = NO DISABLE_INDEX = NO GENERATE_TREEVIEW = YES FULL_SIDEBAR = NO ENUM_VALUES_PER_LINE = 4 TREEVIEW_WIDTH = 250 EXT_LINKS_IN_WINDOW = NO OBFUSCATE_EMAILS = YES HTML_FORMULA_FORMAT = png FORMULA_FONTSIZE = 10 FORMULA_MACROFILE = USE_MATHJAX = NO MATHJAX_VERSION = MathJax_2 MATHJAX_FORMAT = HTML-CSS MATHJAX_RELPATH = MATHJAX_EXTENSIONS = MATHJAX_CODEFILE = SEARCHENGINE = YES SERVER_BASED_SEARCH = NO EXTERNAL_SEARCH = NO SEARCHENGINE_URL = SEARCHDATA_FILE = searchdata.xml EXTERNAL_SEARCH_ID = EXTRA_SEARCH_MAPPINGS = #--------------------------------------------------------------------------- # Configuration options related to the LaTeX output #--------------------------------------------------------------------------- GENERATE_LATEX = NO #--------------------------------------------------------------------------- # Configuration options related to the RTF output #--------------------------------------------------------------------------- GENERATE_RTF = NO #--------------------------------------------------------------------------- # Configuration options related to the man page output #--------------------------------------------------------------------------- GENERATE_MAN = NO #--------------------------------------------------------------------------- # Configuration options related to the XML output #--------------------------------------------------------------------------- GENERATE_XML = NO #--------------------------------------------------------------------------- # Configuration options related to the DOCBOOK output #--------------------------------------------------------------------------- GENERATE_DOCBOOK = NO #--------------------------------------------------------------------------- # Configuration options for the AutoGen Definitions output #--------------------------------------------------------------------------- GENERATE_AUTOGEN_DEF = NO #--------------------------------------------------------------------------- # Configuration options related to Sqlite3 output #--------------------------------------------------------------------------- GENERATE_SQLITE3 = NO #--------------------------------------------------------------------------- # Configuration options related to the Perl module output #--------------------------------------------------------------------------- GENERATE_PERLMOD = NO #--------------------------------------------------------------------------- # Configuration options related to the preprocessor #--------------------------------------------------------------------------- ENABLE_PREPROCESSING = YES MACRO_EXPANSION = YES EXPAND_ONLY_PREDEF = NO SEARCH_INCLUDES = YES INCLUDE_PATH = include INCLUDE_FILE_PATTERNS = PREDEFINED = ZXC_EXPORT= \ ZXC_NO_EXPORT= \ ZXC_DEPRECATED= \ __attribute__(x)= EXPAND_AS_DEFINED = SKIP_FUNCTION_MACROS = YES #--------------------------------------------------------------------------- # Configuration options related to external references #--------------------------------------------------------------------------- TAGFILES = GENERATE_TAGFILE = ALLEXTERNALS = NO EXTERNAL_GROUPS = YES EXTERNAL_PAGES = YES #--------------------------------------------------------------------------- # Configuration options related to diagram generator tools #--------------------------------------------------------------------------- HIDE_UNDOC_RELATIONS = YES HAVE_DOT = NO DOT_NUM_THREADS = 0 DOT_COMMON_ATTR = "fontname=Helvetica,fontsize=10" DOT_EDGE_ATTR = "labelfontname=Helvetica,labelfontsize=10" DOT_NODE_ATTR = "shape=box,height=0.2,width=0.4" DOT_FONTPATH = CLASS_GRAPH = YES COLLABORATION_GRAPH = YES GROUP_GRAPHS = YES UML_LOOK = NO UML_LIMIT_NUM_FIELDS = 10 DOT_UML_DETAILS = NO DOT_WRAP_THRESHOLD = 17 TEMPLATE_RELATIONS = NO INCLUDE_GRAPH = YES INCLUDED_BY_GRAPH = YES CALL_GRAPH = NO CALLER_GRAPH = NO GRAPHICAL_HIERARCHY = YES DIRECTORY_GRAPH = YES DIR_GRAPH_MAX_DEPTH = 1 DOT_IMAGE_FORMAT = png INTERACTIVE_SVG = NO DOT_PATH = DOTFILE_DIRS = DIA_PATH = DIAFILE_DIRS = PLANTUML_JAR_PATH = PLANTUML_CFG_FILE = PLANTUML_INCLUDE_PATH = DOT_GRAPH_MAX_NODES = 50 MAX_DOT_GRAPH_DEPTH = 0 DOT_MULTI_TARGETS = NO GENERATE_LEGEND = YES DOT_CLEANUP = YES MSCGEN_TOOL = MSCFILE_DIRS = zxc-0.11.0/LICENSE000066400000000000000000000063351520102567100134100ustar00rootroot00000000000000============================================================================== ZXC Copyright (c) 2025-2026, Bertrand Lebonnois and contributors License: BSD 3-Clause ============================================================================== BSD 3-Clause License ==================== Copyright (c) 2025-2026, Bertrand Lebonnois and contributors All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of ZXC nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL BERTRAND LEBONNOIS OR CONTRIBUTORS BE LIABLE FOR DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ============================================================================== This project includes code from rapidhash Copyright (C) 2025 Nicolas De Carli License: MIT (Expat) ============================================================================== rapidhash - Very fast, high quality, platform-independent hashing algorithm. Copyright (C) 2025 Nicolas De Carli Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. You can contact the author at: - rapidhash source repository: https://github.com/Nicoshev/rapidhash zxc-0.11.0/Makefile000066400000000000000000000057111520102567100140400ustar00rootroot00000000000000# ZXC - High-performance lossless compression # # Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. # SPDX-License-Identifier: BSD-3-Clause # Top-level convenience Makefile (wraps CMake) # # Usage: # make Build the library + CLI (Release) # make test Build and run tests (parallel) # make format Format source code with clang-format # make format-check Check formatting (CI mode) # make lint Scan source files for non-ASCII characters (CI mirror) # make doc Generate Doxygen documentation # make clean Remove build directory # # Override build directory: make BUILD=mybuild # Pass extra CMake flags: make CMAKE_EXTRA="-DZXC_NATIVE_ARCH=OFF" BUILD ?= build CMAKE ?= cmake CMAKE_EXTRA ?= JOBS ?= $(shell nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) .PHONY: all test format format-check lint doc clean # ── Build ──────────────────────────────────────────────────── all: @$(CMAKE) -S . -B $(BUILD) -DCMAKE_BUILD_TYPE=Release $(CMAKE_EXTRA) @$(CMAKE) --build $(BUILD) -j$(JOBS) # ── Test ───────────────────────────────────────────────────── test: @$(CMAKE) -S . -B $(BUILD) -DCMAKE_BUILD_TYPE=Release -DZXC_BUILD_TESTS=ON $(CMAKE_EXTRA) @$(CMAKE) --build $(BUILD) -j$(JOBS) @cd $(BUILD) && ctest --output-on-failure -j$(JOBS) # ── Formatting ─────────────────────────────────────────────── format: @$(CMAKE) -S . -B $(BUILD) @$(CMAKE) --build $(BUILD) --target format format-check: @$(CMAKE) -S . -B $(BUILD) @$(CMAKE) --build $(BUILD) --target format-check # ── Lint (mirrors .github/workflows/quality.yml) ───────────── # Scans .c and .h files under src/, include/, tests/ for non-ASCII bytes. # Uses Perl for portability. lint: @echo "Scanning for non-ASCII characters in .c and .h files..." @files=$$(find src include tests -type f \( -name '*.c' -o -name '*.h' \) 2>/dev/null); \ if [ -z "$$files" ]; then echo "No source files found."; exit 0; fi; \ LC_ALL=C perl -ne \ 'if (/[^[:ascii:]]/) { print "$$ARGV:$$.:$$_"; $$bad=1 } \ END { exit($$bad ? 1 : 0) }' \ $$files \ && echo "OK: No non-ASCII characters found." \ || { echo "ERROR: Non-ASCII characters found in source files."; exit 1; } # ── Documentation ──────────────────────────────────────────── doc: @$(CMAKE) -S . -B $(BUILD) @$(CMAKE) --build $(BUILD) --target doc # ── Clean ──────────────────────────────────────────────────── clean: @rm -rf $(BUILD) zxc-0.11.0/README.md000066400000000000000000000747171520102567100136730ustar00rootroot00000000000000# ZXC: High-Performance Asymmetric Lossless Compression [![Build & Release](https://github.com/hellobertrand/zxc/actions/workflows/build.yml/badge.svg)](https://github.com/hellobertrand/zxc/actions/workflows/build.yml) [![Code Quality](https://github.com/hellobertrand/zxc/actions/workflows/quality.yml/badge.svg)](https://github.com/hellobertrand/zxc/actions/workflows/quality.yml) [![Fuzzing](https://github.com/hellobertrand/zxc/actions/workflows/fuzzing.yml/badge.svg)](https://github.com/hellobertrand/zxc/actions/workflows/fuzzing.yml) [![Benchmark](https://github.com/hellobertrand/zxc/actions/workflows/benchmark.yml/badge.svg)](https://github.com/hellobertrand/zxc/actions/workflows/benchmark.yml) [![Code Security](https://github.com/hellobertrand/zxc/actions/workflows/security.yml/badge.svg)](https://github.com/hellobertrand/zxc/actions/workflows/security.yml) [![Code Coverage](https://codecov.io/github/hellobertrand/zxc/branch/main/graph/badge.svg?token=LHA03HOA1X)](https://codecov.io/github/hellobertrand/zxc) [![ConanCenter](https://repology.org/badge/version-for-repo/conancenter/zxc.svg)](https://repology.org/project/zxc/versions) [![Vcpkg](https://repology.org/badge/version-for-repo/vcpkg/zxc.svg)](https://repology.org/project/zxc/versions) [![Homebrew](https://repology.org/badge/version-for-repo/homebrew/zxc.svg)](https://repology.org/project/zxc/versions) [![Debian 14](https://repology.org/badge/version-for-repo/debian_14/zxc.svg)](https://repology.org/project/zxc/versions) [![Ubuntu 26.04](https://repology.org/badge/version-for-repo/ubuntu_26_04/zxc.svg)](https://repology.org/project/zxc/versions) [![Crates.io](https://img.shields.io/crates/v/zxc-compress)](https://crates.io/crates/zxc-compress) [![PyPi](https://img.shields.io/pypi/v/zxc-compress)](https://pypi.org/project/zxc-compress) [![npm](https://img.shields.io/npm/v/zxc-compress)](https://www.npmjs.com/package/zxc-compress) [![License](https://img.shields.io/badge/license-BSD--3--Clause-blue)](LICENSE) **ZXC** is a high-performance, lossless, asymmetric compression library that trades compress speed for **maximum decode throughput**: ideal for Content Delivery, Embedded Systems, FOTA (Firmware Over-The-Air) updates, Game Assets, and App Bundles. It follows a **"Write Once, Read Many"** *(WORM)* design: compression happens at build-time, decompression is hot-path at run-time. **ZXC runs on all major architectures** (x86_64, ARM64, ARMv7, ARMv6, RISC-V, POWER (ppc64el), s390x, i386) with hand-tuned SIMD paths (AVX2/AVX-512 on x86_64, NEON on ARMv8+). It shows especially strong gains on modern ARM cores (Apple Silicon, AWS Graviton, Google Axion) thanks to a bitstream layout tuned for their deep pipelines. ## TL;DR - **What:** A C library for lossless compression, optimized for **maximum decompression speed**. - **Key Result:** Up to **>40% faster** decompression than LZ4 on Apple Silicon, **>20% faster** on Google Axion (ARM64), **>10% faster** on x86_64 (AMD EPYC), **all with better compression ratios**. Cross-platform by design, with particularly strong results on ARMv8+. - **Use Cases:** Game assets, firmware, app bundles, anything *compressed once, decompressed millions of times*. - **Seekable:** Built-in seek table for **O(1) random-access** decompression, load any block without scanning the entire file. - **Install:** `conan install --requires="zxc/[*]"` · `vcpkg install zxc` · `brew install zxc` · `pip install zxc-compress` · `cargo add zxc-compress` · `npm i zxc-compress` - **Quality:** Fuzzed (5B+ iterations to date), sanitized, formally tested, thread-safe API. BSD-3-Clause. > **Independently Verified:** ZXC has been officially merged into both major open-source compression benchmark suites: > > - **[lzbench](https://github.com/inikep/lzbench)** (master branch, by @inikep) > - **[TurboBench](https://github.com/powturbo/TurboBench)** (master branch, by @powturbo) > > You can reproduce these results independently using either industry-standard benchmark, alongside 70+ other codecs. ## ZXC Design Philosophy Traditional codecs force a trade-off between **symmetric speed** (LZ4) and **archival density** (Zstd). **ZXC takes a third path: Asymmetric Efficiency.** The encoder performs heavy analysis upfront: match selection, optimal parsing, statistics tuning, to produce a bitstream structured for the instruction pipelining and branch prediction of modern CPUs (particularly ARMv8). Complexity is **offloaded from the decoder to the encoder**, which is exactly the trade-off WORM workloads want. * **Build Time:** You generally compress only once (on CI/CD). * **Run Time:** You decompress millions of times (on every user's device). **ZXC respects this asymmetry.** [👉 **Read the Technical Whitepaper**](docs/WHITEPAPER.md) ## Benchmarks To ensure consistent performance, benchmarks are automatically executed on every commit via GitHub Actions. We monitor metrics on both **x86_64** (Linux) and **ARM64** (Apple Silicon M2) runners to track compression speed, decompression speed, and ratios. *(See the [latest benchmark logs](https://github.com/hellobertrand/zxc/actions/workflows/benchmark.yml))* *Decompression Speed vs Compressed Size — ARM64 Apple M2* ![Decompression Speed vs Compressed Size](docs/images/bench-arm64.svg) ### 1. Mobile & Client: Apple Silicon (M2) *Scenario: Game Assets loading, App startup.* | Target | ZXC vs Competitor | Decompression Speed | Ratio | Verdict | | :--- | :--- | :--- | :--- | :--- | | **1. Max Speed** | **ZXC -1** vs *LZ4 --fast* | **12,530 MB/s** vs 5,623 MB/s **2.23x Faster** | **61.5** vs 62.2 **Smaller** (-0.7%) | **ZXC** leads in raw throughput. | | **2. Standard** | **ZXC -3** vs *LZ4 Default* | **7,049 MB/s** vs 4,783 MB/s **1.47x Faster** | **45.8** vs 47.6 **Smaller** (-1.8%) | **ZXC** outperforms LZ4 in read speed and ratio. | | **3. Max Density** | **ZXC -6** vs *LZ4HC -9* | **5,620 MB/s** vs 4,528 MB/s **1.24x Faster** | **36.3** vs 36.8 **Smaller** (-0.5%) | **ZXC** beats LZ4HC on both decode speed and ratio. | ### 2. Cloud Server: Google Axion (ARM Neoverse V2) *Scenario: High-throughput Microservices, ARM Cloud Instances.* | Target | ZXC vs Competitor | Decompression Speed | Ratio | Verdict | | :--- | :--- | :--- | :--- | :--- | | **1. Max Speed** | **ZXC -1** vs *LZ4 --fast* | **9,067 MB/s** vs 4,951 MB/s **1.83x Faster** | **61.5** vs 62.2 **Smaller** (-0.7%) | **ZXC** leads in raw throughput. | | **2. Standard** | **ZXC -3** vs *LZ4 Default* | **5,297 MB/s** vs 4,259 MB/s **1.24x Faster** | **45.8** vs 47.6 **Smaller** (-1.8%) | **ZXC** outperforms LZ4 in read speed and ratio. | | **3. Max Density** | **ZXC -6** vs *LZ4HC -9* | **4,205 MB/s** vs 3,849 MB/s **1.09x Faster** | **36.3** vs 36.8 **Smaller** (-0.5%) | **ZXC** beats LZ4HC on both decode speed and ratio. | ### 3. Build Server: x86_64 (AMD EPYC 9B45 / Zen 5) *Scenario: CI/CD Pipelines compatibility.* | Target | ZXC vs Competitor | Decompression Speed | Ratio | Verdict | | :--- | :--- | :--- | :--- | :--- | | **1. Max Speed** | **ZXC -1** vs *LZ4 --fast* | **10,844 MB/s** vs 5,301 MB/s **2.05x Faster** | **61.5** vs 62.2 **Smaller** (-0.7%) | **ZXC** achieves higher throughput. | | **2. Standard** | **ZXC -3** vs *LZ4 Default* | **5,955 MB/s** vs 5,013 MB/s **1.19x Faster** | **45.8** vs 47.6 **Smaller** (-1.8%) | **ZXC** offers improved speed and ratio. | | **3. Max Density** | **ZXC -6** vs *LZ4HC -9* | 4,695 MB/s vs **4,841 MB/s** (decode within 3%) | **36.3** vs 36.8 **Smaller** (-0.5%) | **ZXC** wins on ratio; decode trails `LZ4HC -9` by ~3%. | ### 4. Production Server: x86_64 (AMD EPYC 7763 / Zen 3) *Scenario: Mainstream cloud workloads (AWS c6a, Azure HBv3, GCP n2d).* | Target | ZXC vs Competitor | Decompression Speed | Ratio | Verdict | | :--- | :--- | :--- | :--- | :--- | | **1. Max Speed** | **ZXC -1** vs *LZ4 --fast* | **7,077 MB/s** vs 4,092 MB/s **1.73x Faster** | **61.5** vs 62.2 **Smaller** (-0.7%) | **ZXC** holds a strong lead on the legacy x86 pipeline. | | **2. Standard** | **ZXC -3** vs *LZ4 Default* | **3,922 MB/s** vs 3,546 MB/s **1.11x Faster** | **45.8** vs 47.6 **Smaller** (-1.8%) | **ZXC** delivers faster decode and smaller output. | | **3. Max Density** | **ZXC -6** vs *LZ4HC -9* | 3,196 MB/s vs **3,401 MB/s** (decode within 6%) | **36.3** vs 36.8 **Smaller** (-0.5%) | **ZXC** wins on ratio; decode trails `LZ4HC -9` by ~6% on Zen 3. | *Decompression Speed: ZXC vs LZ4 family at equivalent ratio tiers, across 4 CPUs (Fast ≈ 62%, Default ≈ 47%, High ≈ 37%)* ![Decompression Speed: ZXC vs LZ4 family at equivalent ratio tiers](docs/images/bench-bars.svg) *Effective Throughput : Ratio-Normalized Decode across ARM64 and x86 (decode x 100 / ratio%, LZ4 baseline = 1.00x)* ![Effective Throughput](docs/images/bench-effective.svg) > **What is Effective Throughput?** > > Raw decode speed misses half the picture: in real workloads (asset streaming, container pulls, microservice payloads), the decoder is fed by a compressed-byte source - disk, network, inter-core - whose bandwidth is the bottleneck. The right question is *how much original data is delivered per MB of compressed input*. > > Formula: `Effective (MB/s) = Decode × 100 / Ratio (%)`: combines decode speed and ratio in one number. **Every ZXC level sits above LZ4** on every architecture, peaking at **2.0x on Apple Silicon** and ranging **1.15x–1.70x** on x86 and ARM cloud platforms. ### Benchmark ARM64 (Apple Silicon M2) Benchmarks were conducted using lzbench 2.2.1 (from @inikep), compiled with Clang 21.0.0 using *MOREFLAGS="-march=native"* on macOS Tahoe 26.4 (Build 25E246). The reference hardware is an Apple M2 processor (ARM64). All performance metrics reflect single-threaded execution on the standard Silesia Corpus and the benchmark made use of [silesia.tar](https://github.com/DataCompression/corpus-collection/tree/main/Silesia-Corpus), which contains tarred files from the Silesia compression corpus. | Compressor name | Compression| Decompress.| Compr. size | Ratio | Filename | | --------------- | -----------| -----------| ----------- | ----- | -------- | | memcpy | 52866 MB/s | 52887 MB/s | 211947520 |100.00 | 1 files| | **zxc 0.11.0 -1** | 876 MB/s | **12530 MB/s** | 130356444 | **61.50** | 1 files| | **zxc 0.11.0 -2** | 586 MB/s | **10360 MB/s** | 113634139 | **53.61** | 1 files| | **zxc 0.11.0 -3** | 253 MB/s | **7049 MB/s** | 97051816 | **45.79** | 1 files| | **zxc 0.11.0 -4** | 174 MB/s | **6697 MB/s** | 90393215 | **42.65** | 1 files| | **zxc 0.11.0 -5** | 102 MB/s | **6267 MB/s** | 85341643 | **40.27** | 1 files| | **zxc 0.11.0 -6** | 11.8 MB/s | **5620 MB/s** | 76888252 | **36.28** | 1 files| | lz4 1.10.0 | 813 MB/s | 4783 MB/s | 100880800 | 47.60 | 1 files| | lz4 1.10.0 --fast -17 | 1350 MB/s | 5623 MB/s | 131732802 | 62.15 | 1 files| | lz4hc 1.10.0 -9 | 48.2 MB/s | 4528 MB/s | 77884448 | 36.75 | 1 files| | lzav 5.7 -1 | 665 MB/s | 3877 MB/s | 84644732 | 39.94 | 1 files| | snappy 1.2.2 | 880 MB/s | 3264 MB/s | 101415443 | 47.85 | 1 files| | zstd 1.5.7 --fast --1 | 724 MB/s | 2538 MB/s | 86916294 | 41.01 | 1 files| | zstd 1.5.7 -1 | 645 MB/s | 1806 MB/s | 73193704 | 34.53 | 1 files| | zlib 1.3.1 -1 | 150 MB/s | 410 MB/s | 77259029 | 36.45 | 1 files| ### Benchmark ARM64 (Google Axion Neoverse-V2) Benchmarks were conducted using lzbench 2.2.1 (from @inikep), compiled with GCC 14.3.0 using *MOREFLAGS="-march=native"* on Linux 64-bits Debian GNU/Linux 12 (bookworm). The reference hardware is a Google Neoverse-V2 processor (ARM64). All performance metrics reflect single-threaded execution on the standard Silesia Corpus and the benchmark made use of [silesia.tar](https://github.com/DataCompression/corpus-collection/tree/main/Silesia-Corpus), which contains tarred files from the Silesia compression corpus. | Compressor name | Compression| Decompress.| Compr. size | Ratio | Filename | | --------------- | -----------| -----------| ----------- | ----- | -------- | | memcpy | 24179 MB/s | 24134 MB/s | 211947520 |100.00 | 1 files| | **zxc 0.11.0 -1** | 868 MB/s | **9067 MB/s** | 130356444 | **61.50** | 1 files| | **zxc 0.11.0 -2** | 586 MB/s | **7524 MB/s** | 113634139 | **53.61** | 1 files| | **zxc 0.11.0 -3** | 238 MB/s | **5297 MB/s** | 97051816 | **45.79** | 1 files| | **zxc 0.11.0 -4** | 165 MB/s | **5025 MB/s** | 90393215 | **42.65** | 1 files| | **zxc 0.11.0 -5** | 96.9 MB/s | **4685 MB/s** | 85341643 | **40.27** | 1 files| | **zxc 0.11.0 -6** | 11.0 MB/s | **4205 MB/s** | 76888252 | **36.28** | 1 files| | lz4 1.10.0 | 732 MB/s | 4259 MB/s | 100880800 | 47.60 | 1 files| | lz4 1.10.0 --fast -17 | 1280 MB/s | 4951 MB/s | 131732802 | 62.15 | 1 files| | lz4hc 1.10.0 -9 | 43.4 MB/s | 3849 MB/s | 77884448 | 36.75 | 1 files| | lzav 5.7 -1 | 562 MB/s | 2757 MB/s | 84644732 | 39.94 | 1 files| | snappy 1.2.2 | 757 MB/s | 2313 MB/s | 101415443 | 47.85 | 1 files| | zstd 1.5.7 --fast --1 | 607 MB/s | 2295 MB/s | 86916294 | 41.01 | 1 files| | zstd 1.5.7 -1 | 525 MB/s | 1645 MB/s | 73193704 | 34.53 | 1 files| | zlib 1.3.1 -1 | 115 MB/s | 390 MB/s | 77259029 | 36.45 | 1 files| ### Benchmark x86_64 (AMD EPYC 9B45) Benchmarks were conducted using lzbench 2.2.1 (from @inikep), compiled with GCC 14.3.0 using *MOREFLAGS="-march=native"* on Linux 64-bits Ubuntu 24.04. The reference hardware is an AMD EPYC 9B45 processor (x86_64). All performance metrics reflect single-threaded execution on the standard Silesia Corpus and the benchmark made use of [silesia.tar](https://github.com/DataCompression/corpus-collection/tree/main/Silesia-Corpus), which contains tarred files from the Silesia compression corpus. | Compressor name | Compression| Decompress.| Compr. size | Ratio | Filename | | --------------- | -----------| -----------| ----------- | ----- | -------- | | memcpy | 23351 MB/s | 23292 MB/s | 211947520 |100.00 | 1 files| | **zxc 0.11.0 -1** | 859 MB/s | **10844 MB/s** | 130356444 | **61.50** | 1 files| | **zxc 0.11.0 -2** | 584 MB/s | **9597 MB/s** | 113634139 | **53.61** | 1 files| | **zxc 0.11.0 -3** | 238 MB/s | **5955 MB/s** | 97051816 | **45.79** | 1 files| | **zxc 0.11.0 -4** | 163 MB/s | **5589 MB/s** | 90393215 | **42.65** | 1 files| | **zxc 0.11.0 -5** | 97.0 MB/s | **5259 MB/s** | 85341643 | **40.27** | 1 files| | **zxc 0.11.0 -6** | 11.7 MB/s | **4695 MB/s** | 76888252 | **36.28** | 1 files| | lz4 1.10.0 | 767 MB/s | 5013 MB/s | 100880800 | 47.60 | 1 files| | lz4 1.10.0 --fast -17 | 1280 MB/s | 5301 MB/s | 131732802 | 62.15 | 1 files| | lz4hc 1.10.0 -9 | 45.0 MB/s | 4841 MB/s | 77884448 | 36.75 | 1 files| | lzav 5.7 -1 | 600 MB/s | 3628 MB/s | 84644732 | 39.94 | 1 files| | snappy 1.2.2 | 768 MB/s | 2118 MB/s | 101512076 | 47.89 | 1 files| | zstd 1.5.7 --fast --1 | 656 MB/s | 2407 MB/s | 86916294 | 41.01 | 1 files| | zstd 1.5.7 -1 | 597 MB/s | 1868 MB/s | 73193704 | 34.53 | 1 files| | zlib 1.3.1 -1 | 133 MB/s | 387 MB/s | 77259029 | 36.45 | 1 files| ### Benchmark x86_64 (AMD EPYC 7763) Benchmarks were conducted using lzbench 2.2.1 (from @inikep), compiled with GCC 14.2.0 using *MOREFLAGS="-march=native"* on Linux 64-bits Ubuntu 24.04. The reference hardware is an AMD EPYC 7763 64-Core processor (x86_64). All performance metrics reflect single-threaded execution on the standard Silesia Corpus and the benchmark made use of [silesia.tar](https://github.com/DataCompression/corpus-collection/tree/main/Silesia-Corpus), which contains tarred files from the Silesia compression corpus. | Compressor name | Compression| Decompress.| Compr. size | Ratio | Filename | | --------------- | -----------| -----------| ----------- | ----- | -------- | | memcpy | 23023 MB/s | 23087 MB/s | 211947520 |100.00 | 1 files| | **zxc 0.11.0 -1** | 640 MB/s | **7077 MB/s** | 130356444 | **61.50** | 1 files| | **zxc 0.11.0 -2** | 431 MB/s | **5907 MB/s** | 113634139 | **53.61** | 1 files| | **zxc 0.11.0 -3** | 185 MB/s | **3922 MB/s** | 97051816 | **45.79** | 1 files| | **zxc 0.11.0 -4** | 128 MB/s | **3775 MB/s** | 90393215 | **42.65** | 1 files| | **zxc 0.11.0 -5** | 76.5 MB/s | **3624 MB/s** | 85341643 | **40.27** | 1 files| | **zxc 0.11.0 -6** | 8.85 MB/s | **3196 MB/s** | 76888252 | **36.28** | 1 files| | lz4 1.10.0 | 580 MB/s | 3546 MB/s | 100880800 | 47.60 | 1 files| | lz4 1.10.0 --fast -17 | 1015 MB/s | 4092 MB/s | 131732802 | 62.15 | 1 files| | lz4hc 1.10.0 -9 | 33.8 MB/s | 3401 MB/s | 77884448 | 36.75 | 1 files| | lzav 5.7 -1 | 407 MB/s | 2609 MB/s | 84644732 | 39.94 | 1 files| | snappy 1.2.2 | 612 MB/s | 1591 MB/s | 101512076 | 47.89 | 1 files| | zstd 1.5.7 --fast --1 | 443 MB/s | 1626 MB/s | 86916294 | 41.01 | 1 files| | zstd 1.5.7 -1 | 400 MB/s | 1221 MB/s | 73193704 | 34.53 | 1 files| | zlib 1.3.1 -1 | 98.1 MB/s | 328 MB/s | 77259029 | 36.45 | 1 files| --- ## Installation ### Option 1: Download Release (GitHub) 1. Go to the [Releases page](https://github.com/hellobertrand/zxc/releases). 2. Download the archive matching your architecture: **macOS:** * `zxc-macos-arm64.tar.gz` (NEON optimizations included). **Linux:** * `zxc-linux-aarch64.tar.gz` (NEON optimizations included). * `zxc-linux-x86_64.tar.gz` (Runtime dispatch for AVX2/AVX512). **Windows:** * `zxc-windows-x64.zip` (Runtime dispatch for AVX2/AVX512). * `zxc-windows-arm64.zip` (NEON optimizations included). 3. Extract and install: ```bash tar -xzf zxc-linux-x86_64.tar.gz -C /usr/local ``` Each archive contains: ``` bin/zxc # CLI binary include/ # C headers (zxc.h, zxc_buffer.h, ...) lib/libzxc.a # Static library lib/pkgconfig/libzxc.pc # pkg-config support lib/cmake/zxc/zxcConfig.cmake # CMake find_package(zxc) support ``` 4. Use in your project: **CMake:** ```cmake find_package(zxc REQUIRED) target_link_libraries(myapp PRIVATE zxc::zxc_lib) ``` **pkg-config:** ```bash cc myapp.c $(pkg-config --cflags --libs libzxc) -o myapp ``` ### Option 2: vcpkg **Classic mode:** ```bash vcpkg install zxc ``` **Manifest mode** (add to `vcpkg.json`): ```json { "dependencies": ["zxc"] } ``` Then in your CMake project: ```cmake find_package(zxc CONFIG REQUIRED) target_link_libraries(myapp PRIVATE zxc::zxc_lib) ``` ### Option 3: Conan You also can download and install zxc using the [Conan](https://conan.io/) package manager: ```bash conan install -r conancenter --requires="zxc/[*]" --build=missing ``` Or add to your `conanfile.txt`: ```ini [requires] zxc/[*] ``` The zxc package in Conan Center is kept up to date by [ConanCenterIndex](https://github.com/conan-io/conan-center-index) contributors. If the version is out of date, please create an issue or pull request on the Conan Center Index repository. ### Option 4: Homebrew ```bash brew install zxc ``` The formula is maintained in [homebrew-core](https://formulae.brew.sh/formula/zxc). ### Option 5: Building from Source **Requirements:** CMake (3.14+), C17 Compiler (Clang/GCC/MSVC). ```bash git clone https://github.com/hellobertrand/zxc.git cd zxc cmake -B build -DCMAKE_BUILD_TYPE=Release cmake --build build --parallel # Run tests ctest --test-dir build -C Release --output-on-failure # CLI usage ./build/zxc --help # Install library, headers, and CMake/pkg-config files sudo cmake --install build ``` #### CMake Options | Option | Default | Description | |--------|---------|-------------| | `BUILD_SHARED_LIBS` | OFF | Build shared libraries instead of static (`libzxc.so`, `libzxc.dylib`, `zxc.dll`) | | `ZXC_NATIVE_ARCH` | ON | Enable `-march=native` for maximum performance | | `ZXC_ENABLE_LTO` | ON | Enable Link-Time Optimization (LTO) | | `ZXC_PGO_MODE` | OFF | Profile-Guided Optimization mode (`OFF`, `GENERATE`, `USE`) | | `ZXC_BUILD_CLI` | ON | Build command-line interface | | `ZXC_BUILD_TESTS` | ON | Build unit tests | | `ZXC_ENABLE_COVERAGE` | OFF | Enable code coverage generation (disables LTO/PGO) | | `ZXC_DISABLE_SIMD` | OFF | Disable hand-written SIMD paths (AVX2/AVX512/NEON) | ```bash # Build shared library cmake -B build -DBUILD_SHARED_LIBS=ON # Portable build (without -march=native) cmake -B build -DZXC_NATIVE_ARCH=OFF # Library only (no CLI, no tests) cmake -B build -DZXC_BUILD_CLI=OFF -DZXC_BUILD_TESTS=OFF # Code coverage build cmake -B build -DZXC_ENABLE_COVERAGE=ON # Disable explicit SIMD code paths (compiler auto-vectorisation is unaffected) cmake -B build -DZXC_DISABLE_SIMD=ON ``` #### Profile-Guided Optimization (PGO) PGO uses runtime profiling data to optimize branch layout, inlining decisions, and code placement. **Step 1 - Build with instrumentation:** ```bash cmake -B build -DCMAKE_BUILD_TYPE=Release -DZXC_PGO_MODE=GENERATE cmake --build build --parallel ``` **Step 2 - Run a representative workload to collect profile data:** ```bash # Run the test suite (exercises all block types and compression levels) ./build/zxc_test # Or compress/decompress representative data ./build/zxc -b your_data_file ``` **Step 3 - (Clang only) Merge raw profiles:** ```bash # Clang generates .profraw files that must be merged before use llvm-profdata merge -output=build/pgo/default.profdata build/pgo/*.profraw ``` > GCC uses a directory-based format and does not require this step. **Step 4 - Rebuild with profile data:** ```bash cmake -B build -DCMAKE_BUILD_TYPE=Release -DZXC_PGO_MODE=USE cmake --build build --parallel ``` ### Packaging Status [![Packaging status](https://repology.org/badge/vertical-allrepos/zxc.svg)](https://repology.org/project/zxc/versions) --- ## Compression Levels * **Level 1, 2 (Fast):** Optimized for real-time assets (Gaming, UI). * **Level 3, 4 (Balanced):** A strong middle-ground offering efficient compression speed and a ratio superior to LZ4. * **Level 5 (Compact):** A good choice for Embedded and Firmware. Better compression than LZ4 and significantly faster decoding than Zstd. * **Level 6 (Max):** Highest ratio tier, matching LZ4-HC while keeping ZXC's decode advantage. Best for Archival and write-once / read-many workloads where compression time is amortized over many reads. ## Block Size Tuning The default block size is **512 KB**, tuned for bulk/archival workloads where ratio and decompression throughput matter most. For **memory-constrained or streaming use cases**, **256 KB blocks** halve the per-context memory footprint at a small cost in ratio and decompression speed. **Why larger blocks help:** Each block starts with a cold hash table, so the LZ match-finder has no history and produces more literals until the table warms up. Doubling the block size halves the number of cold-start penalties, improving both ratio and decompression speed. | Block Size | cctx memory | dctx memory | Ratio (level -3) | Decompression gain vs 256 KB | |:----------:|:-----------:|:-----------:|:----------------:|:----------------------------:| | 256 KB | ~1.03 MB | ~256 KB | 46.36% | — | | 512 KB *(default)* | ~1.78 MB | ~512 KB | 45.81% *(−0.55 pp)* | +1% to +8% depending on CPU | ```bash # CLI — fall back to 256 KB blocks (e.g. embedded / streaming) zxc -B 256K -5 input_file output_file # API zxc_compress_opts_t opts = { .level = ZXC_LEVEL_COMPACT, .block_size = 256 * 1024, }; ``` **Guideline:** Stick with 512 KB (default) for bulk compression pipelines, CI/CD asset packaging, and high-throughput servers. Use 256 KB (`-B 256K`) for streaming, embedded, or memory-constrained environments. --- ## Usage ### 1. CLI The CLI is perfect for benchmarking or manually compressing assets. ```bash # Basic Compression (Level 3 is default) zxc -z input_file output_file # High Compression (Level 5) zxc -z -5 input_file output_file # Seekable Archive (enables O(1) random-access decompression) zxc -z -S input_file output_file # -z for compression can be omitted zxc input_file output_file # as well as output file; it will be automatically assigned to input_file.zxc zxc input_file # Decompression zxc -d compressed_file output_file # Benchmark Mode (Testing speed on your machine) zxc -b input_file ``` #### Using with `tar` ZXC works as a drop-in external compressor for `tar` (reads stdin, writes stdout, returns 0 on success): ```bash # GNU tar (Linux) tar -I 'zxc -5' -cf archive.tar.zxc data/ tar -I 'zxc -d' -xf archive.tar.zxc # bsdtar (macOS) tar --use-compress-program='zxc -5' -cf archive.tar.zxc data/ tar --use-compress-program='zxc -d' -xf archive.tar.zxc # Pipes (universal) tar cf - data/ | zxc > archive.tar.zxc zxc -d < archive.tar.zxc | tar xf - ``` ### 2. API ZXC provides a **thread-safe API** with two usage patterns. Parameters are passed through dedicated options structs, making call sites self-documenting and forward-compatible. #### Buffer API (In-Memory) ```c #include "zxc.h" // Compression uint64_t bound = zxc_compress_bound(src_size); zxc_compress_opts_t c_opts = { .level = ZXC_LEVEL_DEFAULT, .checksum_enabled = 1, /* .block_size = 0 -> 512 KB default */ }; int64_t compressed_size = zxc_compress(src, src_size, dst, bound, &c_opts); // Decompression zxc_decompress_opts_t d_opts = { .checksum_enabled = 1 }; int64_t decompressed_size = zxc_decompress(src, src_size, dst, dst_capacity, &d_opts); ``` #### Stream API (Files, Multi-Threaded) ```c #include "zxc.h" // Compression (auto-detect threads, level 3, checksum on) zxc_compress_opts_t c_opts = { .n_threads = 0, // 0 = auto .level = ZXC_LEVEL_DEFAULT, .checksum_enabled = 1, /* .block_size = 0 -> 512 KB default */ }; int64_t bytes_written = zxc_stream_compress(f_in, f_out, &c_opts); // Decompression zxc_decompress_opts_t d_opts = { .n_threads = 0, .checksum_enabled = 1 }; int64_t bytes_out = zxc_stream_decompress(f_in, f_out, &d_opts); ``` #### Reusable Context API (Low-Latency / Embedded) For tight loops (e.g. filesystem plug-ins) where per-call `malloc`/`free` overhead matters, use opaque reusable contexts. Options are **sticky** - settings from `zxc_create_cctx()` are reused when passing `NULL`: ```c #include "zxc.h" zxc_compress_opts_t opts = { .level = 3, .checksum_enabled = 0 }; zxc_cctx* cctx = zxc_create_cctx(&opts); // allocate once, settings remembered zxc_dctx* dctx = zxc_create_dctx(); // allocate once // reuse across many blocks - NULL reuses sticky settings: int64_t csz = zxc_compress_cctx(cctx, src, src_sz, dst, dst_cap, NULL); int64_t dsz = zxc_decompress_dctx(dctx, dst, csz, out, src_sz, NULL); zxc_free_cctx(cctx); zxc_free_dctx(dctx); ``` **Features:** - Caller-allocated buffers with explicit bounds - Thread-safe (stateless) - Configurable block sizes (4 KB – 2 MB, powers of 2) - Multi-threaded streaming (auto-detects CPU cores) - Optional checksum validation - Reusable contexts for high-frequency call sites - Seekable archives: optional seek table for O(1) random-access decompression (`.seekable = 1`) **[👉 See complete examples and advanced usage](docs/EXAMPLES.md)** ## Language Bindings [![Crates.io](https://img.shields.io/crates/v/zxc-compress)](https://crates.io/crates/zxc-compress) [![PyPi](https://img.shields.io/pypi/v/zxc-compress)](https://pypi.org/project/zxc-compress) [![npm](https://img.shields.io/npm/v/zxc-compress)](https://www.npmjs.com/package/zxc-compress) Official wrappers maintained in this repository: | Language | Package Manager | Install Command | Documentation | Author | |----------|-----------------|-----------------|---------------|--------| | **Rust** | [`crates.io`](https://crates.io/crates/zxc-compress) | `cargo add zxc-compress` | [README](wrappers/rust/zxc/README.md) | [@hellobertrand](https://github.com/hellobertrand) | | **Python**| [`PyPI`](https://pypi.org/project/zxc-compress) | `pip install zxc-compress` | [README](wrappers/python/README.md) | [@nuberchardzer1](https://github.com/nuberchardzer1) | | **Node.js**| [`npm`](https://www.npmjs.com/package/zxc-compress) | `npm install zxc-compress` | [README](wrappers/nodejs/README.md) | [@hellobertrand](https://github.com/hellobertrand) | | **Go** | `go get` | `go get github.com/hellobertrand/zxc/wrappers/go` | [README](wrappers/go/README.md) | [@hellobertrand](https://github.com/hellobertrand) | | **WASM** | Build from source | `emcmake cmake -B build-wasm && cmake --build build-wasm` | [README](wrappers/wasm/README.md) | [@hellobertrand](https://github.com/hellobertrand) | Community-maintained bindings: | Language | Package Manager | Install Command | Repository | Author | | -------- | --------------- | --------------- | ---------- | ------ | | **Go** | pkg.go.dev | `go get github.com/meysam81/go-zxc` | | [@meysam81](https://github.com/meysam81) | | **Nim** | nimble | `nimble install zxc` | | [@georgelemon](https://github.com/georgelemon) | | **Free Pascal** | Build from source | Clone the repository | | [@Xelitan](https://github.com/Xelitan) | ## Safety & Quality * **Unit Tests**: Comprehensive test suite with CTest integration. * **Continuous Fuzzing**: Integrated with ClusterFuzzLite suites — **5+ billion iterations** accumulated to date across compress, decompress, streaming and seekable API surfaces. * **Static Analysis**: Checked with Cppcheck & Clang Static Analyzer. * **CodeQL Analysis**: GitHub Advanced Security scanning for vulnerabilities. * **Snyk**: Continuous security and code analysis for dependencies and source. * **Code Coverage**: Automated tracking with Codecov integration. * **Dynamic Analysis**: Validated with Valgrind and ASan/UBSan in CI pipelines. * **Safe API**: Explicit buffer capacity is required for all operations. ## License & Credits **ZXC** Copyright © 2025-2026, Bertrand Lebonnois and contributors. Licensed under the **BSD 3-Clause License**. See LICENSE for details. **Third-Party Components:** - **rapidhash** by Nicolas De Carli (MIT) - Used for high-speed, platform-independent checksums. zxc-0.11.0/codecov.yml000066400000000000000000000001361520102567100145410ustar00rootroot00000000000000coverage: status: project: default: target: 80% ignore: - "src/cli/**" zxc-0.11.0/docs/000077500000000000000000000000001520102567100133245ustar00rootroot00000000000000zxc-0.11.0/docs/API.md000066400000000000000000001107051520102567100142630ustar00rootroot00000000000000# ZXC API & ABI Reference **Library version**: 0.11.0 **SOVERSION**: 3 **License**: BSD-3-Clause This document is the authoritative reference for the public API surface and ABI guarantees of **libzxc**. It is intended for integrators, packagers, and language-binding authors. For usage examples see [`EXAMPLES.md`](EXAMPLES.md). For the on-disk binary format see [`FORMAT.md`](FORMAT.md). --- ## Table of Contents - [1. Headers and Include Graph](#1-headers-and-include-graph) - [2. Symbol Visibility](#2-symbol-visibility) - [3. ABI Versioning](#3-abi-versioning) - [4. Runtime Dependencies](#4-runtime-dependencies) - [5. Constants and Enumerations](#5-constants-and-enumerations) - [6. Type Definitions](#6-type-definitions) - [7. Buffer API](#7-buffer-api) - [8. Block API](#8-block-api) - [9. Reusable Context API](#9-reusable-context-api) - [10. Streaming API](#10-streaming-api) - [10b. Push Streaming API](#10b-push-streaming-api) - [11. Seekable API](#11-seekable-api) - [12. Sans-IO API](#12-sans-io-api) - [13. Error Handling](#13-error-handling) - [14. Thread Safety](#14-thread-safety) - [15. Exported Symbols Summary](#15-exported-symbols-summary) --- ## 1. Headers and Include Graph ```text zxc.h <- umbrella header (includes everything below) ├── zxc_buffer.h <- Buffer API + Reusable Context API │ ├── zxc_export.h <- visibility macros │ └── zxc_stream.h <- Streaming API + opts structs │ └── zxc_export.h ├── zxc_constants.h <- version macros, compression levels, block sizes ├── zxc_error.h <- error codes + zxc_error_name() │ └── zxc_export.h ├── zxc_pstream.h <- Push streaming API (caller-driven, single-thread) │ ├── zxc_export.h │ └── zxc_stream.h └── (not included by zxc.h) zxc_seekable.h <- Seekable random-access API (opt-in) └── zxc_export.h zxc_sans_io.h <- Low-level primitives (opt-in) └── zxc_export.h ``` Include `` to access everything except the sans-IO and seekable layers. Include `` explicitly for random-access decompression. Include `` explicitly when building custom drivers. --- ## 2. Symbol Visibility libzxc uses an **opt-in** export strategy: | Build type | How symbols are exposed | |-----------|------------------------| | **Shared library** | Default visibility is `hidden`. Only functions annotated with `ZXC_EXPORT` are exported. Internal FMV variants (`_default`, `_neon`, `_avx2`, `_avx512`) are hidden. | | **Static library** | Define `ZXC_STATIC_DEFINE` (set automatically by CMake) to disable import/export annotations. | ### Macros | Macro | Purpose | |-------|---------| | `ZXC_EXPORT` | Marks a symbol as part of the public API (`__declspec(dllexport/dllimport)` on Windows, `visibility("default")` on GCC/Clang). | | `ZXC_NO_EXPORT` | Forces a symbol to be hidden (`visibility("hidden")`). | | `ZXC_DEPRECATED` | Emits a compiler warning when a deprecated symbol is used. | | `ZXC_STATIC_DEFINE` | Define when building or consuming as a static library. | | `zxc_lib_EXPORTS` | Set automatically by CMake when building the shared library. Do not define manually. | --- ## 3. ABI Versioning libzxc follows the shared-library versioning convention: ``` libzxc.so.{SOVERSION}.{MAJOR}.{MINOR}.{PATCH} ``` | Field | Description | Current | |-------|-------------|---------| | `SOVERSION` | Bumped on **ABI-breaking** changes (struct layout, removed symbols, changed signatures). | **3** | | `VERSION` | Tracks the library release. | **0.11.0** | **Compatibility rule**: any binary compiled against SOVERSION N will load against any libzxc with the same SOVERSION, regardless of the `VERSION` triple. ### Platform naming | Platform | Files | |----------|-------| | Linux | `libzxc.so` -> `libzxc.so.3` -> `libzxc.so.0.11.0` | | macOS | `libzxc.dylib` -> `libzxc.3.dylib` -> `libzxc.0.11.0.dylib` | | Windows | `zxc.dll` + `zxc.lib` (import) | --- ## 4. Runtime Dependencies libzxc has **zero external dependencies**. | Dependency | Notes | |-----------|-------| | C standard library | ``, ``, ``, `` | | POSIX threads | `pthread` on Unix/macOS; Windows threads on Win32 | No dependency on OpenSSL, zlib, or any other compression library. --- ## 5. Constants and Enumerations ### 5.1 Version (compile-time) Defined in `zxc_constants.h`: ```c #define ZXC_VERSION_MAJOR 0 #define ZXC_VERSION_MINOR 10 #define ZXC_VERSION_PATCH 0 #define ZXC_LIB_VERSION_STR "0.11.0" ``` ### 5.2 Block Size Constraints ```c #define ZXC_BLOCK_SIZE_MIN_LOG2 12 // exponent for minimum #define ZXC_BLOCK_SIZE_MAX_LOG2 21 // exponent for maximum #define ZXC_BLOCK_SIZE_MIN (1U << 12) // 4 KB #define ZXC_BLOCK_SIZE_MAX (1U << 21) // 2 MB #define ZXC_BLOCK_SIZE_DEFAULT (512 * 1024) // 512 KB ``` Block size must be a power of two within `[ZXC_BLOCK_SIZE_MIN, ZXC_BLOCK_SIZE_MAX]`. Pass `0` to any API to use `ZXC_BLOCK_SIZE_DEFAULT`. ### 5.3 Compression Levels ```c typedef enum { ZXC_LEVEL_FASTEST = 1, // Best throughput, lowest ratio ZXC_LEVEL_FAST = 2, // Fast, good for real-time ZXC_LEVEL_DEFAULT = 3, // Recommended balance ZXC_LEVEL_BALANCED = 4, // Higher ratio, good speed ZXC_LEVEL_COMPACT = 5, // High density ZXC_LEVEL_DENSITY = 6 // Maximum density: Huffman literals + optimal parser } zxc_compression_level_t; ``` Levels 1..5 produce data decompressible at essentially the **same speed**; level 6 adds a per-block Huffman decode step that costs ~10–20 % decode throughput compared to lower levels in exchange for the densest output. Pass `0` for level to use `ZXC_LEVEL_DEFAULT`. ### 5.4 Error Codes ```c typedef enum { ZXC_OK = 0, ZXC_ERROR_MEMORY = -1, // malloc failure ZXC_ERROR_DST_TOO_SMALL = -2, // output buffer too small ZXC_ERROR_SRC_TOO_SMALL = -3, // input truncated ZXC_ERROR_BAD_MAGIC = -4, // invalid magic word ZXC_ERROR_BAD_VERSION = -5, // unsupported format version ZXC_ERROR_BAD_HEADER = -6, // corrupted header (CRC mismatch) ZXC_ERROR_BAD_CHECKSUM = -7, // checksum verification failed ZXC_ERROR_CORRUPT_DATA = -8, // corrupted compressed data ZXC_ERROR_BAD_OFFSET = -9, // invalid match offset ZXC_ERROR_OVERFLOW = -10, // buffer overflow detected ZXC_ERROR_IO = -11, // file read/write/seek failure ZXC_ERROR_NULL_INPUT = -12, // required pointer is NULL ZXC_ERROR_BAD_BLOCK_TYPE = -13, // unknown block type ZXC_ERROR_BAD_BLOCK_SIZE = -14 // invalid block size } zxc_error_t; ``` All public functions that can fail return negative `zxc_error_t` values on error. --- ## 6. Type Definitions ### 6.1 Options Structs **Compression options** (defined in `zxc_stream.h`, used by all compression functions): ```c typedef struct { int n_threads; // Worker thread count (0 = auto-detect). int level; // Compression level 1–6 (0 = default). size_t block_size; // Block size in bytes (0 = 512 KB default). int checksum_enabled; // 1 = enable checksums, 0 = disable. int seekable; // 1 = append seek table for random access. zxc_progress_callback_t progress_cb; // Optional callback (NULL to disable). void* user_data; // Passed through to progress_cb. } zxc_compress_opts_t; ``` **Decompression options**: ```c typedef struct { int n_threads; // Worker thread count (0 = auto-detect). int checksum_enabled; // 1 = verify checksums, 0 = skip. zxc_progress_callback_t progress_cb; // Optional callback. void* user_data; // Passed through to progress_cb. } zxc_decompress_opts_t; ``` Both structs are safe to zero-initialize for default behavior. Pass `NULL` instead of an options pointer to use all defaults. ### 6.2 Progress Callback ```c typedef void (*zxc_progress_callback_t)( uint64_t bytes_processed, // Total input bytes processed so far uint64_t bytes_total, // Total input bytes (0 if unknown, e.g. stdin) const void* user_data // User-provided context ); ``` Called from the writer thread after each block is processed. Must be fast and non-blocking. ### 6.3 Opaque Context Types ```c typedef struct zxc_cctx_s zxc_cctx; // Opaque compression context typedef struct zxc_dctx_s zxc_dctx; // Opaque decompression context ``` Internal layout is hidden. Interact only through `zxc_create_*` / `zxc_free_*` / `zxc_*_cctx` / `zxc_*_dctx` functions. ### 6.4 Sans-IO Types **Compression context** (public struct, for advanced use only): ```c typedef struct { uint32_t* hash_table; // LZ77 hash table uint16_t* chain_table; // Collision chain table void* memory_block; // Single allocation owner uint32_t epoch; // Lazy hash invalidation counter uint32_t* buf_sequences; // Packed sequence records uint8_t* buf_tokens; // Token buffer uint16_t* buf_offsets; // Offset buffer uint8_t* buf_extras; // Extra-length buffer uint8_t* literals; // Literal bytes uint8_t* lit_buffer; // Scratch buffer for RLE size_t lit_buffer_cap; uint8_t* work_buf; // Padded scratch for buffer-API decompression size_t work_buf_cap; int checksum_enabled; int compression_level; size_t chunk_size; // Effective block size uint32_t offset_bits; // log2(chunk_size) uint32_t offset_mask; // (1 << offset_bits) - 1 uint32_t max_epoch; // 1 << (32 - offset_bits) } zxc_cctx_t; ``` **Block header** (8 bytes on disk): ```c typedef struct { uint8_t block_type; // See FORMAT.md §4 uint8_t block_flags; uint8_t reserved; uint8_t header_crc; // 1-byte header CRC uint32_t comp_size; // Compressed payload size (excl. header) } zxc_block_header_t; ``` --- ## 7. Buffer API Declared in `zxc_buffer.h`. Single-threaded, blocking, in-memory operations. ### Library Info Helpers Runtime-queryable library metadata. Exposed so integrations and language bindings can discover the supported level range and library version without relying on compile-time constants alone. #### `zxc_min_level` ```c ZXC_EXPORT int zxc_min_level(void); ``` Returns the minimum supported compression level (currently `1`, equivalent to `ZXC_LEVEL_FASTEST`). #### `zxc_max_level` ```c ZXC_EXPORT int zxc_max_level(void); ``` Returns the maximum supported compression level (currently `5`, equivalent to `ZXC_LEVEL_COMPACT`). #### `zxc_default_level` ```c ZXC_EXPORT int zxc_default_level(void); ``` Returns the default compression level (currently `3`, equivalent to `ZXC_LEVEL_DEFAULT`). #### `zxc_version_string` ```c ZXC_EXPORT const char* zxc_version_string(void); ``` Returns the library version as a null-terminated string (e.g. `"0.11.0"`). The returned pointer is a compile-time constant and must not be freed. ### `zxc_compress_bound` ```c ZXC_EXPORT uint64_t zxc_compress_bound(size_t input_size); ``` Returns the worst-case compressed size for `input_size` bytes. Use to allocate the destination buffer before compression. ### `zxc_compress` ```c ZXC_EXPORT int64_t zxc_compress( const void* src, size_t src_size, void* dst, size_t dst_capacity, const zxc_compress_opts_t* opts // NULL = defaults ); ``` Compresses `src` into `dst`. Only `level`, `block_size`, `checksum_enabled`, and `seekable` fields of `opts` are used. `n_threads` is ignored (always single-threaded). **Returns**: compressed size (> 0) on success, or negative `zxc_error_t`. ### `zxc_decompress` ```c ZXC_EXPORT int64_t zxc_decompress( const void* src, size_t src_size, void* dst, size_t dst_capacity, const zxc_decompress_opts_t* opts // NULL = defaults ); ``` Decompresses `src` into `dst`. Only `checksum_enabled` is used. **Returns**: decompressed size (> 0) on success, or negative `zxc_error_t`. ### `zxc_get_decompressed_size` ```c ZXC_EXPORT uint64_t zxc_get_decompressed_size( const void* src, size_t src_size ); ``` Reads the original size from the file footer without decompressing. **Returns**: original size, or `0` if the buffer is invalid. --- ## 8. Block API Declared in `zxc_buffer.h`. Single-block compression and decompression **without file framing** (no file header, EOF block, or footer). Designed for filesystem integrations (DwarFS, EROFS, SquashFS) where the caller manages its own block indexing. Output format: `block_header (8 B)` + compressed payload + optional `checksum (4 B)`. ### `zxc_compress_block_bound` ```c ZXC_EXPORT uint64_t zxc_compress_block_bound(size_t input_size); ``` Returns the maximum compressed size for a single block. Unlike `zxc_compress_bound()`, this does **not** include file header, EOF block, or footer overhead. **Returns**: upper bound in bytes, or `0` on overflow. ### `zxc_decompress_block_bound` ```c ZXC_EXPORT uint64_t zxc_decompress_block_bound(size_t uncompressed_size); ``` Returns the minimum `dst_capacity` required by `zxc_decompress_block()` for a block of `uncompressed_size` bytes. The fast decoder uses speculative wild-copy writes and needs a small tail pad beyond the declared uncompressed size: passing exactly `uncompressed_size` as `dst_capacity` forces the slow tail path and may trigger `ZXC_ERROR_OVERFLOW` on some inputs. Use this helper to size destination buffers for the fast path. For callers that genuinely cannot oversize their output buffer, use `zxc_decompress_block_safe()` instead. **Returns**: minimum `dst_capacity` in bytes, or `0` if `uncompressed_size` would overflow. ### `zxc_compress_block` ```c ZXC_EXPORT int64_t zxc_compress_block( zxc_cctx* cctx, const void* src, size_t src_size, void* dst, size_t dst_capacity, const zxc_compress_opts_t* opts // NULL = defaults ); ``` Compresses a single block using a reusable context. Only `level`, `block_size`, and `checksum_enabled` fields of `opts` are used. **Returns**: compressed block size (> 0) on success, or negative `zxc_error_t`. ### `zxc_decompress_block` ```c ZXC_EXPORT int64_t zxc_decompress_block( zxc_dctx* dctx, const void* src, size_t src_size, void* dst, size_t dst_capacity, const zxc_decompress_opts_t* opts // NULL = defaults ); ``` Decompresses a single block produced by `zxc_compress_block()`. `dst_capacity` should be at least `zxc_decompress_block_bound(uncompressed_size)` to enable the fast path. Only `checksum_enabled` is used. **Returns**: decompressed size (> 0) on success, or negative `zxc_error_t`. ### `zxc_decompress_block_safe` ```c ZXC_EXPORT int64_t zxc_decompress_block_safe( zxc_dctx* dctx, const void* src, size_t src_size, void* dst, size_t dst_capacity, const zxc_decompress_opts_t* opts // NULL = defaults ); ``` Strict-sized variant of `zxc_decompress_block()`. Accepts `dst_capacity == uncompressed_size` exactly: no tail pad required. Intended for integrations whose output buffer cannot be oversized (page-aligned decoding into mapped pages, fixed-size slots in a columnar layout, etc.). Output is **bit-identical** to `zxc_decompress_block()`. NUM and RAW blocks transparently forward to the fast path; only GLO/GHI blocks use the strict-tail decoder, which is slightly slower than the wild-copy fast path (see the performance table in `EXAMPLES.md`). Only `checksum_enabled` is used. **Returns**: decompressed size (> 0) on success, or negative `zxc_error_t`. ### `zxc_estimate_cctx_size` ```c ZXC_EXPORT uint64_t zxc_estimate_cctx_size(size_t src_size, int level); ``` Returns an accurate estimate of the peak memory used when compressing a single block of `src_size` bytes at the given `level` via `zxc_compress_block()`. The estimate covers all per-chunk working buffers (chain table, literals, sequence/token/offset/extras buffers) plus the fixed hash tables and the cache-line alignment padding. At `level >= 6` it also includes the transient DP scratch (~18 × `src_size` bytes) malloc'd by the price-based optimal parser for the duration of each block. It scales roughly linearly with `src_size` and is intended for integrators that need to build an accurate memory budget (filesystems, embedded devices, sandboxed workloads). **Returns**: estimated peak cctx memory usage in bytes, or `0` if `src_size == 0`. --- ## 9. Reusable Context API Declared in `zxc_buffer.h`. Eliminates per-call allocation overhead for hot-path integrations (filesystem plug-ins, batch processing). ### `zxc_create_cctx` ```c ZXC_EXPORT zxc_cctx* zxc_create_cctx(const zxc_compress_opts_t* opts); ``` Creates a heap-allocated compression context. When `opts` is non-NULL, internal buffers are pre-allocated (eager init). When `opts` is NULL, allocation is deferred to first use (lazy init). **Returns**: context pointer, or `NULL` on allocation failure. ### `zxc_free_cctx` ```c ZXC_EXPORT void zxc_free_cctx(zxc_cctx* cctx); ``` Frees all resources. Safe to pass `NULL`. ### `zxc_compress_cctx` ```c ZXC_EXPORT int64_t zxc_compress_cctx( zxc_cctx* cctx, const void* src, size_t src_size, void* dst, size_t dst_capacity, const zxc_compress_opts_t* opts ); ``` Same as `zxc_compress()` but reuses internal buffers from `cctx`. Automatically re-initializes when `block_size` or `level` changes. ### `zxc_create_dctx` ```c ZXC_EXPORT zxc_dctx* zxc_create_dctx(void); ``` Creates a heap-allocated decompression context. ### `zxc_free_dctx` ```c ZXC_EXPORT void zxc_free_dctx(zxc_dctx* dctx); ``` Frees all resources. Safe to pass `NULL`. ### `zxc_decompress_dctx` ```c ZXC_EXPORT int64_t zxc_decompress_dctx( zxc_dctx* dctx, const void* src, size_t src_size, void* dst, size_t dst_capacity, const zxc_decompress_opts_t* opts ); ``` Same as `zxc_decompress()` but reuses buffers from `dctx`. --- ## 10. Streaming API Declared in `zxc_stream.h`. Multi-threaded, `FILE*`-based pipeline (reader -> workers -> writer). ### `zxc_stream_compress` ```c ZXC_EXPORT int64_t zxc_stream_compress( FILE* f_in, FILE* f_out, const zxc_compress_opts_t* opts ); ``` Compresses `f_in` -> `f_out` using a parallel pipeline. All fields of `opts` are used, including `n_threads` and `progress_cb`. **Returns**: total compressed bytes written, or negative `zxc_error_t`. ### `zxc_stream_decompress` ```c ZXC_EXPORT int64_t zxc_stream_decompress( FILE* f_in, FILE* f_out, const zxc_decompress_opts_t* opts ); ``` Decompresses `f_in` -> `f_out` using a parallel pipeline. **Returns**: total decompressed bytes written, or negative `zxc_error_t`. ### `zxc_stream_get_decompressed_size` ```c ZXC_EXPORT int64_t zxc_stream_get_decompressed_size(FILE* f_in); ``` Reads the original size from the file footer. File position is restored. **Returns**: original size, or negative `zxc_error_t`. --- ## 10b. Push Streaming API Declared in `zxc_pstream.h`. Single-threaded, **caller-driven** streaming — the inverse of the `FILE*`-based pipeline. Designed for callback-based integrations (async runtimes, network protocols) that cannot block on a `FILE*` and need to feed/drain in arbitrary chunks. The on-disk output is **bit-compatible** with the Buffer / Stream APIs: files produced by the push API decode with `zxc_decompress()` / `zxc_stream_decompress()`, and vice-versa. ### Buffer Descriptors ```c typedef struct { const void* src; size_t size; size_t pos; // advanced by the library } zxc_inbuf_t; typedef struct { void* dst; size_t size; size_t pos; // advanced by the library } zxc_outbuf_t; ``` The library reads `[src+pos .. src+size)` and writes `[dst+pos .. dst+size)`, advancing `pos` by what it consumed/produced. Buffers are caller-owned. ### Compression #### `zxc_cstream_create` ```c ZXC_EXPORT zxc_cstream* zxc_cstream_create(const zxc_compress_opts_t* opts); ``` Creates a push compression context. Settings (`level`, `block_size`, `checksum_enabled`) are copied from `opts` and frozen for the lifetime of the stream. `n_threads`, `progress_cb`, `seekable` are ignored on this path. **Returns**: context, or `NULL` on allocation failure. #### `zxc_cstream_free` ```c ZXC_EXPORT void zxc_cstream_free(zxc_cstream* cs); ``` Releases all resources. Safe with `NULL`. #### `zxc_cstream_compress` ```c ZXC_EXPORT int64_t zxc_cstream_compress( zxc_cstream* cs, zxc_outbuf_t* out, zxc_inbuf_t* in ); ``` Pushes input into the stream and drains compressed output: - emits the file header on the first call; - copies input into the internal block accumulator; - compresses one block whenever the accumulator fills, writing it into `out` (up to `out->size`); - returns when `in` is fully consumed *and* no more compressed bytes are pending, or when `out` has no room left. Fully reentrant: if `out` fills mid-block, the next call resumes draining where it left off. Safe to call with empty input (drain-only). **Returns**: - `0` — `in` fully consumed and no pending output; - `>0` — bytes still pending in the staging buffer (drain `out` and call again); - `<0` — `zxc_error_t` (sticky: subsequent calls return the same code). #### `zxc_cstream_end` ```c ZXC_EXPORT int64_t zxc_cstream_end(zxc_cstream* cs, zxc_outbuf_t* out); ``` Finalises the stream: compresses any partial last block, emits the EOF block (8 B) and the file footer (12 B). **Must be called** to produce a valid ZXC file. Reentrant the same way `_compress` is: loop until it returns `0`. After `_end` returns `0`, the stream is in DONE state and any further call returns `ZXC_ERROR_NULL_INPUT`. **Returns**: - `0` — finalisation complete; - `>0` — bytes still pending (drain `out` and call again); - `<0` — `zxc_error_t`. #### `zxc_cstream_in_size` / `zxc_cstream_out_size` ```c ZXC_EXPORT size_t zxc_cstream_in_size(const zxc_cstream* cs); ZXC_EXPORT size_t zxc_cstream_out_size(const zxc_cstream* cs); ``` Suggested buffer sizes for best throughput. The caller may use any size; these are purely performance hints. ### Decompression #### `zxc_dstream_create` ```c ZXC_EXPORT zxc_dstream* zxc_dstream_create(const zxc_decompress_opts_t* opts); ``` Creates a push decompression context. Only `checksum_enabled` from `opts` is honoured (controls whether the global file-level checksum is verified when the file carries one). **Returns**: context, or `NULL` on allocation failure. #### `zxc_dstream_free` ```c ZXC_EXPORT void zxc_dstream_free(zxc_dstream* ds); ``` Releases all resources. Safe with `NULL`. #### `zxc_dstream_decompress` ```c ZXC_EXPORT int64_t zxc_dstream_decompress( zxc_dstream* ds, zxc_outbuf_t* out, zxc_inbuf_t* in ); ``` Drives the parser state machine (file header → blocks → EOF → optional SEK → footer). Each call makes as much progress as `in` and `out` allow. Trailing bytes after the validated footer are silently ignored (the caller can inspect `in->pos` to detect how many were consumed). **Returns**: - `>0` — number of decompressed bytes written into `out` this call; - `0` — either DONE (use `zxc_dstream_finished` to confirm) or no progress possible without more input; - `<0` — `zxc_error_t` (sticky). #### `zxc_dstream_finished` ```c ZXC_EXPORT int zxc_dstream_finished(const zxc_dstream* ds); ``` Returns `1` iff the parser has fully validated the file footer. Callers that have finished feeding input should check this to detect truncated streams: `zxc_dstream_decompress` returning `0` with no output is ambiguous (DONE vs need-more-input) — `_finished` disambiguates. #### `zxc_dstream_in_size` / `zxc_dstream_out_size` ```c ZXC_EXPORT size_t zxc_dstream_in_size(const zxc_dstream* ds); ZXC_EXPORT size_t zxc_dstream_out_size(const zxc_dstream* ds); ``` Suggested buffer sizes. Returns `0` if `ds` is `NULL`. Before the file header has been parsed, `zxc_dstream_in_size()` may return a default recommended input size hint (for example `ZXC_BLOCK_SIZE_DEFAULT`), because the actual block size is not known until it is learned from the header. ### Threading Each `zxc_cstream` / `zxc_dstream` is single-threaded: one context, one thread. Multiple contexts may be used concurrently from different threads. ### Compression Example ```c zxc_compress_opts_t opts = { .level = 3, .checksum_enabled = 1 }; zxc_cstream* cs = zxc_cstream_create(&opts); uint8_t in_buf[64*1024], out_buf[64*1024]; zxc_outbuf_t out = { out_buf, sizeof out_buf, 0 }; ssize_t n; while ((n = read_some(in_buf, sizeof in_buf)) > 0) { zxc_inbuf_t in = { in_buf, (size_t)n, 0 }; while (in.pos < in.size) { int64_t r = zxc_cstream_compress(cs, &out, &in); if (r < 0) goto fatal; if (out.pos > 0) { write_to_sink(out_buf, out.pos); out.pos = 0; } } } int64_t pending; do { pending = zxc_cstream_end(cs, &out); if (pending < 0) goto fatal; if (out.pos > 0) { write_to_sink(out_buf, out.pos); out.pos = 0; } } while (pending > 0); zxc_cstream_free(cs); ``` --- ## 11. Seekable API Declared in `zxc_seekable.h` (not included by `zxc.h`: opt-in, optional). Random-access decompression of seekable archives produced with `seekable = 1`. ### Creating a Seekable Archive Set `seekable = 1` in `zxc_compress_opts_t`. Works with both the Buffer API and the Streaming API: ```c zxc_compress_opts_t opts = { .level = 3, .seekable = 1 }; int64_t csize = zxc_compress(src, src_size, dst, dst_cap, &opts); ``` The resulting archive contains a Seek Table block (SEK) between the EOF block and the file footer. Standard decompressors handle seekable archives transparently, the seek table is skipped during sequential decompression. ### `zxc_seekable_open` ```c ZXC_EXPORT zxc_seekable* zxc_seekable_open(const void* src, size_t src_size); ``` Opens a seekable archive from a memory buffer. The buffer must remain valid for the lifetime of the handle. **Returns**: handle on success, or `NULL` if the buffer is not a valid seekable archive. ### `zxc_seekable_open_file` ```c ZXC_EXPORT zxc_seekable* zxc_seekable_open_file(FILE* f); ``` Opens a seekable archive from a `FILE*`. The file must be seekable (not stdin/pipe). The file position is saved and restored after parsing. **Returns**: handle on success, or `NULL` on error. ### `zxc_seekable_get_num_blocks` ```c ZXC_EXPORT uint32_t zxc_seekable_get_num_blocks(const zxc_seekable* s); ``` Returns the total number of data blocks in the archive. ### `zxc_seekable_get_decompressed_size` ```c ZXC_EXPORT uint64_t zxc_seekable_get_decompressed_size(const zxc_seekable* s); ``` Returns the total decompressed size of the archive. ### `zxc_seekable_get_block_comp_size` ```c ZXC_EXPORT uint32_t zxc_seekable_get_block_comp_size( const zxc_seekable* s, uint32_t block_idx ); ``` Returns the compressed size (on-disk, including header) of a specific block. ### `zxc_seekable_get_block_decomp_size` ```c ZXC_EXPORT uint32_t zxc_seekable_get_block_decomp_size( const zxc_seekable* s, uint32_t block_idx ); ``` Returns the decompressed size of a specific block. ### `zxc_seekable_decompress_range` ```c ZXC_EXPORT int64_t zxc_seekable_decompress_range( zxc_seekable* s, void* dst, size_t dst_capacity, uint64_t offset, size_t len ); ``` Decompresses `len` bytes starting at byte `offset` in the original uncompressed data. Only the blocks overlapping the requested range are read and decompressed. **Returns**: `len` on success, or negative `zxc_error_t`. ### `zxc_seekable_decompress_range_mt` ```c ZXC_EXPORT int64_t zxc_seekable_decompress_range_mt( zxc_seekable* s, void* dst, size_t dst_capacity, uint64_t offset, size_t len, int n_threads // 0 = auto-detect ); ``` Multi-threaded variant. Each worker thread uses `pread()` (POSIX) or `ReadFile()` (Windows) for lock-free concurrent I/O. Falls back to single-threaded mode when `n_threads <= 1` or the range spans a single block. **Returns**: `len` on success, or negative `zxc_error_t`. ### `zxc_seekable_free` ```c ZXC_EXPORT void zxc_seekable_free(zxc_seekable* s); ``` Frees a seekable handle and all associated resources. Safe to call with `NULL`. ### `zxc_write_seek_table` ```c ZXC_EXPORT int64_t zxc_write_seek_table( uint8_t* dst, size_t dst_capacity, const uint32_t* comp_sizes, uint32_t num_blocks ); ``` Low-level: writes a seek table (block header + entries) to `dst`. **Returns**: bytes written, or negative `zxc_error_t`. ### `zxc_seek_table_size` ```c ZXC_EXPORT size_t zxc_seek_table_size(uint32_t num_blocks); ``` Returns the encoded byte size of a seek table for `num_blocks` blocks. --- ## 12. Sans-IO API Declared in `zxc_sans_io.h` (not included by `zxc.h` - opt-in). Low-level primitives for building custom compression drivers. ### `zxc_cctx_init` ```c ZXC_EXPORT int zxc_cctx_init( zxc_cctx_t* ctx, size_t chunk_size, int mode, // 1 = compression, 0 = decompression int level, int checksum_enabled ); ``` Allocates internal buffers sized for `chunk_size`. **Returns**: `ZXC_OK` or `ZXC_ERROR_MEMORY`. ### `zxc_cctx_free` ```c ZXC_EXPORT void zxc_cctx_free(zxc_cctx_t* ctx); ``` Frees internal buffers. Does **not** free `ctx` itself. ### `zxc_write_file_header` ```c ZXC_EXPORT int zxc_write_file_header( uint8_t* dst, size_t dst_capacity, size_t chunk_size, int has_checksum ); ``` Writes the 16-byte file header. **Returns**: bytes written, or `ZXC_ERROR_DST_TOO_SMALL`. ### `zxc_read_file_header` ```c ZXC_EXPORT int zxc_read_file_header( const uint8_t* src, size_t src_size, size_t* out_block_size, // optional int* out_has_checksum // optional ); ``` Validates the file header (magic, version, CRC16). **Returns**: `ZXC_OK`, or `ZXC_ERROR_BAD_MAGIC` / `ZXC_ERROR_BAD_VERSION` / `ZXC_ERROR_SRC_TOO_SMALL`. ### `zxc_write_block_header` ```c ZXC_EXPORT int zxc_write_block_header( uint8_t* dst, size_t dst_capacity, const zxc_block_header_t* bh ); ``` Serializes a block header (8 bytes, little-endian). **Returns**: bytes written, or `ZXC_ERROR_DST_TOO_SMALL`. ### `zxc_read_block_header` ```c ZXC_EXPORT int zxc_read_block_header( const uint8_t* src, size_t src_size, zxc_block_header_t* bh ); ``` Parses a block header (endianness conversion included). **Returns**: `ZXC_OK` or `ZXC_ERROR_SRC_TOO_SMALL`. ### `zxc_write_file_footer` ```c ZXC_EXPORT int zxc_write_file_footer( uint8_t* dst, size_t dst_capacity, uint64_t src_size, uint32_t global_hash, int checksum_enabled ); ``` Writes the 12-byte footer (original size + optional global hash). **Returns**: bytes written, or `ZXC_ERROR_DST_TOO_SMALL`. --- ## 13. Error Handling ### `zxc_error_name` ```c ZXC_EXPORT const char* zxc_error_name(int code); ``` Returns a constant, null-terminated string for any error code (e.g. `"ZXC_OK"`, `"ZXC_ERROR_MEMORY"`). Returns `"ZXC_UNKNOWN_ERROR"` for unrecognized codes. ### Pattern All functions that can fail use the same convention: ```c int64_t result = zxc_compress(src, src_size, dst, dst_cap, &opts); if (result < 0) { fprintf(stderr, "Error: %s\n", zxc_error_name((int)result)); } ``` --- ## 14. Thread Safety | API Layer | Safe to call concurrently? | Notes | |-----------|---------------------------|-------| | **Buffer API** | Yes (stateless) | Each call is self-contained. Multiple threads can compress/decompress simultaneously with independent buffers. | | **Block API** | Per-context | Uses `zxc_cctx` / `zxc_dctx`: same rule as Context API. Create one context per thread. | | **Context API** | Per-context | A single `zxc_cctx` / `zxc_dctx` must not be shared between threads. Create one context per thread. | | **Streaming API** | Per-call | Each `zxc_stream_*` call manages its own thread pool internally. Do not call from multiple threads on the same `FILE*`. | | **Seekable API** | Per-handle | A single `zxc_seekable` handle must not be shared between threads for single-threaded decompression. Use `zxc_seekable_decompress_range_mt()` for parallel access. | | **Sans-IO API** | Per-context | Same rule as context API: one `zxc_cctx_t` per thread. | | `zxc_error_name` | Yes | Returns a pointer to a static string. | --- ## 15. Exported Symbols Summary The shared library exports exactly **54 symbols** (verified with `nm -gU`): | # | Symbol | API Layer | Header | |---|--------|-----------|--------| | 1 | `zxc_min_level` | Info | `zxc_buffer.h` | | 2 | `zxc_max_level` | Info | `zxc_buffer.h` | | 3 | `zxc_default_level` | Info | `zxc_buffer.h` | | 4 | `zxc_version_string` | Info | `zxc_buffer.h` | | 5 | `zxc_compress_bound` | Buffer | `zxc_buffer.h` | | 6 | `zxc_compress` | Buffer | `zxc_buffer.h` | | 7 | `zxc_decompress` | Buffer | `zxc_buffer.h` | | 8 | `zxc_get_decompressed_size` | Buffer | `zxc_buffer.h` | | 9 | `zxc_compress_block_bound` | Block | `zxc_buffer.h` | | 10 | `zxc_decompress_block_bound` | Block | `zxc_buffer.h` | | 11 | `zxc_compress_block` | Block | `zxc_buffer.h` | | 12 | `zxc_decompress_block` | Block | `zxc_buffer.h` | | 13 | `zxc_decompress_block_safe` | Block | `zxc_buffer.h` | | 14 | `zxc_estimate_cctx_size` | Block | `zxc_buffer.h` | | 15 | `zxc_create_cctx` | Context | `zxc_buffer.h` | | 16 | `zxc_free_cctx` | Context | `zxc_buffer.h` | | 17 | `zxc_compress_cctx` | Context | `zxc_buffer.h` | | 18 | `zxc_create_dctx` | Context | `zxc_buffer.h` | | 19 | `zxc_free_dctx` | Context | `zxc_buffer.h` | | 20 | `zxc_decompress_dctx` | Context | `zxc_buffer.h` | | 21 | `zxc_stream_compress` | Streaming | `zxc_stream.h` | | 22 | `zxc_stream_decompress` | Streaming | `zxc_stream.h` | | 23 | `zxc_stream_get_decompressed_size` | Streaming | `zxc_stream.h` | | 24 | `zxc_cstream_create` | Push Streaming | `zxc_pstream.h` | | 25 | `zxc_cstream_free` | Push Streaming | `zxc_pstream.h` | | 26 | `zxc_cstream_compress` | Push Streaming | `zxc_pstream.h` | | 27 | `zxc_cstream_end` | Push Streaming | `zxc_pstream.h` | | 28 | `zxc_cstream_in_size` | Push Streaming | `zxc_pstream.h` | | 29 | `zxc_cstream_out_size` | Push Streaming | `zxc_pstream.h` | | 30 | `zxc_dstream_create` | Push Streaming | `zxc_pstream.h` | | 31 | `zxc_dstream_free` | Push Streaming | `zxc_pstream.h` | | 32 | `zxc_dstream_decompress` | Push Streaming | `zxc_pstream.h` | | 33 | `zxc_dstream_finished` | Push Streaming | `zxc_pstream.h` | | 34 | `zxc_dstream_in_size` | Push Streaming | `zxc_pstream.h` | | 35 | `zxc_dstream_out_size` | Push Streaming | `zxc_pstream.h` | | 36 | `zxc_seekable_open` | Seekable | `zxc_seekable.h` | | 37 | `zxc_seekable_open_file` | Seekable | `zxc_seekable.h` | | 38 | `zxc_seekable_get_num_blocks` | Seekable | `zxc_seekable.h` | | 39 | `zxc_seekable_get_decompressed_size` | Seekable | `zxc_seekable.h` | | 40 | `zxc_seekable_get_block_comp_size` | Seekable | `zxc_seekable.h` | | 41 | `zxc_seekable_get_block_decomp_size` | Seekable | `zxc_seekable.h` | | 42 | `zxc_seekable_decompress_range` | Seekable | `zxc_seekable.h` | | 43 | `zxc_seekable_decompress_range_mt` | Seekable | `zxc_seekable.h` | | 44 | `zxc_seekable_free` | Seekable | `zxc_seekable.h` | | 45 | `zxc_write_seek_table` | Seekable | `zxc_seekable.h` | | 46 | `zxc_seek_table_size` | Seekable | `zxc_seekable.h` | | 47 | `zxc_cctx_init` | Sans-IO | `zxc_sans_io.h` | | 48 | `zxc_cctx_free` | Sans-IO | `zxc_sans_io.h` | | 49 | `zxc_write_file_header` | Sans-IO | `zxc_sans_io.h` | | 50 | `zxc_read_file_header` | Sans-IO | `zxc_sans_io.h` | | 51 | `zxc_write_block_header` | Sans-IO | `zxc_sans_io.h` | | 52 | `zxc_read_block_header` | Sans-IO | `zxc_sans_io.h` | | 53 | `zxc_write_file_footer` | Sans-IO | `zxc_sans_io.h` | | 54 | `zxc_error_name` | Error | `zxc_error.h` | No internal symbols leak into the public ABI. FMV dispatch variants (`_default`, `_neon`, `_avx2`, `_avx512`) are compiled with `-fvisibility=hidden` and are not exported. zxc-0.11.0/docs/EXAMPLES.md000066400000000000000000000256551520102567100151010ustar00rootroot00000000000000# ZXC API Examples This document provides complete, working examples for using the ZXC compression library in C. ## Table of Contents - [Buffer API (In-Memory)](#buffer-api-in-memory) - [Stream API (Multi-Threaded)](#stream-api-multi-threaded) - [Reusable Context API](#reusable-context-api) - [Language Bindings](#language-bindings) --- ## Buffer API (In-Memory) Ideal for small assets or simple integrations. Thread-safe and ready for highly concurrent environments (Go routines, Node.js workers, Python threads). ```c #include "zxc.h" #include #include #include int main(void) { // Original data to compress const char* original = "Hello, ZXC! This is a sample text for compression."; size_t original_size = strlen(original) + 1; // Include null terminator // Step 1: Calculate maximum compressed size uint64_t max_compressed_size = zxc_compress_bound(original_size); // Step 2: Allocate buffers void* compressed = malloc(max_compressed_size); void* decompressed = malloc(original_size); if (!compressed || !decompressed) { fprintf(stderr, "Memory allocation failed\n"); free(compressed); free(decompressed); return 1; } // Step 3: Compress data (level 3, checksum enabled, default block size) zxc_compress_opts_t c_opts = { .level = ZXC_LEVEL_DEFAULT, .checksum_enabled = 1, /* .block_size = 0 -> 512 KB default */ }; int64_t compressed_size = zxc_compress( original, // Source buffer original_size, // Source size compressed, // Destination buffer max_compressed_size, // Destination capacity &c_opts // Options (NULL = all defaults) ); if (compressed_size <= 0) { fprintf(stderr, "Compression failed (error %lld)\n", (long long)compressed_size); free(compressed); free(decompressed); return 1; } printf("Original size: %zu bytes\n", original_size); printf("Compressed size: %lld bytes (%.1f%% ratio)\n", (long long)compressed_size, 100.0 * (double)compressed_size / (double)original_size); // Step 4: Decompress data (checksum verification enabled) zxc_decompress_opts_t d_opts = { .checksum_enabled = 1 }; int64_t decompressed_size = zxc_decompress( compressed, // Source buffer (size_t)compressed_size, decompressed, // Destination buffer original_size, // Destination capacity &d_opts // Options (NULL = all defaults) ); if (decompressed_size <= 0) { fprintf(stderr, "Decompression failed (error %lld)\n", (long long)decompressed_size); free(compressed); free(decompressed); return 1; } // Step 5: Verify integrity if ((size_t)decompressed_size == original_size && memcmp(original, decompressed, original_size) == 0) { printf("Success! Data integrity verified.\n"); printf("Decompressed: %s\n", (char*)decompressed); } else { fprintf(stderr, "Data mismatch after decompression\n"); } // Cleanup free(compressed); free(decompressed); return 0; } ``` **Compilation:** ```bash gcc -o buffer_example buffer_example.c -I include -L build -lzxc_lib ``` --- ## Stream API (Multi-Threaded) For large files, use the streaming API to process data in parallel chunks. ```c #include "zxc.h" #include #include int main(int argc, char* argv[]) { if (argc != 4) { fprintf(stderr, "Usage: %s \n", argv[0]); return 1; } const char* input_path = argv[1]; const char* compressed_path = argv[2]; const char* output_path = argv[3]; /* ------------------------------------------------------------------ */ /* Step 1: Compress */ /* ------------------------------------------------------------------ */ printf("Compressing '%s' to '%s'...\n", input_path, compressed_path); FILE* f_in = fopen(input_path, "rb"); if (!f_in) { fprintf(stderr, "Error: cannot open '%s'\n", input_path); return 1; } FILE* f_out = fopen(compressed_path, "wb"); if (!f_out) { fprintf(stderr, "Error: cannot create '%s'\n", compressed_path); fclose(f_in); return 1; } // 0 threads = auto-detect CPU cores; 0 block_size = 512 KB default zxc_compress_opts_t c_opts = { .n_threads = 0, .level = ZXC_LEVEL_DEFAULT, .checksum_enabled = 1, /* .block_size = 0 -> 512 KB */ }; int64_t compressed_bytes = zxc_stream_compress(f_in, f_out, &c_opts); fclose(f_in); fclose(f_out); if (compressed_bytes < 0) { fprintf(stderr, "Compression failed (error %lld)\n", (long long)compressed_bytes); return 1; } printf("Compression complete: %lld bytes written\n", (long long)compressed_bytes); /* ------------------------------------------------------------------ */ /* Step 2: Decompress */ /* ------------------------------------------------------------------ */ printf("\nDecompressing '%s' to '%s'...\n", compressed_path, output_path); FILE* f_compressed = fopen(compressed_path, "rb"); if (!f_compressed) { fprintf(stderr, "Error: cannot open '%s'\n", compressed_path); return 1; } FILE* f_decompressed = fopen(output_path, "wb"); if (!f_decompressed) { fprintf(stderr, "Error: cannot create '%s'\n", output_path); fclose(f_compressed); return 1; } zxc_decompress_opts_t d_opts = { .n_threads = 0, .checksum_enabled = 1 }; int64_t decompressed_bytes = zxc_stream_decompress(f_compressed, f_decompressed, &d_opts); fclose(f_compressed); fclose(f_decompressed); if (decompressed_bytes < 0) { fprintf(stderr, "Decompression failed (error %lld)\n", (long long)decompressed_bytes); return 1; } printf("Decompression complete: %lld bytes written\n", (long long)decompressed_bytes); printf("\nSuccess! Verify the output file matches the original.\n"); return 0; } ``` **Compilation:** ```bash gcc -o stream_example stream_example.c -I include -L build -lzxc_lib -lpthread -lm ``` **Usage:** ```bash ./stream_example large_file.bin compressed.zxc decompressed.bin ``` **Features demonstrated:** - Multi-threaded parallel processing (auto-detects CPU cores) - Checksum validation for data integrity - Error handling for file operations --- ## Reusable Context API For tight loops - such as filesystem plug-ins (squashfs, dwarfs, erofs) - where allocating and freeing internal buffers on every call would add latency, ZXC provides opaque reusable contexts. Internal buffers are only reallocated when the **block size changes**, so the common case (same block size, different data) is completely allocation-free. Options are **sticky**: settings passed via `opts` to `zxc_create_cctx()` or `zxc_compress_cctx()` are remembered and reused on subsequent calls where `opts` is `NULL`. ```c #include "zxc.h" #include #include #include #define BLOCK_SIZE (64 * 1024) // 64 KB - typical squashfs block size #define NUM_BLOCKS 32 int main(void) { // Create context with sticky options (level 3, no checksum, 64 KB blocks). // These settings are remembered for all subsequent calls. zxc_compress_opts_t create_opts = { .level = 3, .checksum_enabled = 0, .block_size = BLOCK_SIZE, }; zxc_cctx* cctx = zxc_create_cctx(&create_opts); zxc_dctx* dctx = zxc_create_dctx(); if (!cctx || !dctx) { fprintf(stderr, "Context creation failed\n"); zxc_free_cctx(cctx); zxc_free_dctx(dctx); return 1; } const size_t comp_cap = (size_t)zxc_compress_bound(BLOCK_SIZE); uint8_t* comp = malloc(comp_cap); uint8_t* src = malloc(BLOCK_SIZE); uint8_t* dec = malloc(BLOCK_SIZE); for (int i = 0; i < NUM_BLOCKS; i++) { // Fill block with pseudo-random data for (size_t j = 0; j < BLOCK_SIZE; j++) src[j] = (uint8_t)(i ^ j); // Compress - pass NULL to reuse sticky settings from create_opts. // No malloc/free inside, no need to pass opts again. int64_t csz = zxc_compress_cctx(cctx, src, BLOCK_SIZE, comp, comp_cap, NULL); if (csz <= 0) { fprintf(stderr, "Block %d: compress error %lld\n", i, (long long)csz); break; } // Decompress - no malloc/free inside int64_t dsz = zxc_decompress_dctx(dctx, comp, (size_t)csz, dec, BLOCK_SIZE, NULL); if (dsz != BLOCK_SIZE || memcmp(src, dec, BLOCK_SIZE) != 0) { fprintf(stderr, "Block %d: roundtrip mismatch\n", i); break; } } // Override sticky settings for a single call (e.g. switch to level 5). // This also updates the sticky settings for future NULL calls. zxc_compress_opts_t high_opts = { .level = 5 }; int64_t csz = zxc_compress_cctx(cctx, src, BLOCK_SIZE, comp, comp_cap, &high_opts); printf("Level-5 compressed: %lld bytes\n", (long long)csz); printf("Processed %d blocks of %d KB - no per-block allocation.\n", NUM_BLOCKS, BLOCK_SIZE / 1024); zxc_free_cctx(cctx); zxc_free_dctx(dctx); free(src); free(comp); free(dec); return 0; } ``` **Compilation:** ```bash gcc -o ctx_example ctx_example.c -I include -L build -lzxc_lib ``` --- ## Writing Your Own Streaming Driver The streaming multi-threaded API shown above is the default provided driver. However, ZXC is written in a **"sans-IO" style** that separates compute from I/O and multitasking. This allows you to write your own driver in any language of your choice, using the native I/O and multitasking capabilities of that language. To implement a custom driver: 1. Include the extra public header `zxc_sans_io.h` 2. Study the reference implementation in `src/lib/zxc_driver.c` 3. Implement your own I/O and threading logic --- ## Language Bindings For non-C languages, see the official bindings: | Language | Package | Install Command | Documentation | |----------|---------|-----------------|---------------| | **Rust** | [`crates.io`](https://crates.io/crates/zxc-compress) | `cargo add zxc-compress` | [README](../wrappers/rust/zxc/README.md) | | **Python**| [`PyPI`](https://pypi.org/project/zxc-compress) | `pip install zxc-compress` | [README](../wrappers/python/README.md) | | **Node.js**| [`npm`](https://www.npmjs.com/package/zxc-compress) | `npm install zxc-compress` | [README](../wrappers/nodejs/README.md) | | **Go** | `go get` | `go get github.com/hellobertrand/zxc/wrappers/go` | [README](../wrappers/go/README.md) | Community-maintained: - **Go**: https://github.com/meysam81/go-zxc - **Nim**: https://github.com/openpeeps/zxc-nim - **Free Pascal**: https://github.com/Xelitan/Free-Pascal-port-of-ZXC-compressor-decompressor zxc-0.11.0/docs/FORMAT.md000066400000000000000000000551571520102567100146530ustar00rootroot00000000000000# ZXC Compressed File Format (Technical Specification) **Date**: May 2026 **Format Version**: 5 This document describes the on-disk binary format of a ZXC compressed file. It formalizes the current reference implementation of format version **5**. ## 1. Conventions - **Byte order**: all multi-byte integers are **little-endian**. - **Unit**: offsets are in bytes, zero-based from the start of each structure. - **Checksum mode**: enabled globally by a flag in the file header. - **Block model**: a file is a sequence of blocks terminated by an EOF block, then a footer. --- ## 2. Full File Layout ```text +----------------------+ 16 bytes | File Header | +----------------------+ | Block #0 | | - 8B Block Header | | - Block Payload | | - Optional 4B CRC32 | +----------------------+ | Block #1 | | ... | +----------------------+ | EOF Block | 8 bytes (type=255, comp_size=0) +----------------------+ | SEK Block (Optional) | table of contents for random access +----------------------+ | File Footer | 12 bytes +----------------------+ ``` --- ## 3. File Header (16 bytes) ```text Offset Size Field 0x00 4 Magic Word 0x04 1 Format Version 0x05 1 Chunk Size Code 0x06 1 Flags 0x07 7 Reserved (must be 0) 0x0E 2 Header CRC16 ``` ### 3.1 Field definitions - **Magic Word** (`u32`): `0x9CB02EF5`. - **Format Version** (`u8`): currently `5`. - **Chunk Size Code** (`u8`): - If the value is in the range `[12, 21]`, it is an **exponent**: `block_size = 2^code`. - `12` = 4 KB, `13` = 8 KB, ..., `19` = 512 KB (default), ..., `21` = 2 MB. - The legacy value `64` (from older encoders) is accepted and maps to 256 KB. - All other values are rejected (`ZXC_ERROR_BAD_BLOCK_SIZE`). - Valid block sizes are powers of 2 in the range **4 KB – 2 MB**. - **Flags** (`u8`): - Bit 7 (`0x80`): `HAS_CHECKSUM`. - Bits 0..3: checksum algorithm id (`0` = RapidHash-based folding). - Bits 4..6: reserved. - **Reserved**: 7 bytes set to zero. - **Header CRC16** (`u16`): computed with `zxc_hash16` on the 16-byte header where bytes `0x0E..0x0F` are zeroed. --- ## 4. Generic Block Container Each block starts with a fixed 8-byte block header. ```text Offset Size Field 0x00 1 Block Type 0x01 1 Block Flags 0x02 1 Reserved 0x03 4 Compressed Payload Size (comp_size) 0x07 1 Header CRC8 ``` ### 4.1 Header semantics - **Block Type**: - `0` = RAW - `1` = GLO - `2` = NUM - `3` = GHI - `254` = SEK - `255` = EOF - **Block Flags**: currently not used by implementation (written as `0`). - **Reserved**: must be 0. - **comp_size**: payload size in bytes (does **not** include the optional trailing 4-byte block checksum). - **Header CRC8**: `zxc_hash8` over the 8-byte header with byte `0x07` forced to zero before hashing. ### 4.2 Block physical layout ```text [8B Block Header] + [comp_size bytes payload] + [optional 4B checksum] ``` When checksums are enabled at file level, each non-EOF block carries one trailing 4-byte checksum of its compressed payload. --- ## 5. Block Types and Payload Formats ## 5.1 RAW block (`type=0`) Payload is uncompressed data. ```text Payload = raw bytes raw_size = comp_size ``` No internal sub-header. --- ## 5.2 NUM block (`type=2`) Used for numeric data (32-bit integer stream), delta/zigzag + bitpacking. ### NUM payload layout ```text +--------------------------+ | NUM Header (16 bytes) | +--------------------------+ | Frame #0 header (16B) | | Frame #0 packed bits | +--------------------------+ | Frame #1 header (16B) | | Frame #1 packed bits | +--------------------------+ | ... | +--------------------------+ ``` ### NUM Header (16 bytes) ```text Offset Size Field 0x00 8 n_values (u64) 0x08 2 frame_size (u16, currently 128) 0x0A 6 reserved ``` ### NUM frame record (repeated) ```text Offset Size Field 0x00 2 nvals in frame (u16) 0x02 2 bits per value (u16) 0x04 8 base/running seed (u64) 0x0C 4 packed_size in bytes (u32) 0x10 ... packed delta bitstream ``` Notes: - Values are reconstructed by bit-unpacking, zigzag decode, then prefix accumulation. - `packed_size` bytes immediately follow each 16-byte frame header. --- ## 5.3 GLO block (`type=1`) General LZ-style format with separated streams. ### GLO payload layout ```text +-------------------------------+ | GLO Header (16 bytes) | +-------------------------------+ | 4 Section Descriptors (32B) | +-------------------------------+ | Literals stream | +-------------------------------+ | Tokens stream | +-------------------------------+ | Offsets stream | +-------------------------------+ | Extras stream | +-------------------------------+ ``` ### GLO Header (16 bytes) ```text Offset Size Field 0x00 4 n_sequences (u32) 0x04 4 n_literals (u32) 0x08 1 enc_lit (0=RAW, 1=RLE, 2=HUFFMAN) 0x09 1 enc_litlen (reserved) 0x0A 1 enc_mlen (reserved) 0x0B 1 enc_off (0=16-bit offsets, 1=8-bit offsets) 0x0C 4 reserved ``` ### GLO section descriptors (4 × 8 bytes) Descriptor format (packed `u64`): - low 32 bits: compressed size - high 32 bits: raw size Section order: 1. Literals 2. Tokens 3. Offsets 4. Extras ### GLO stream content - **Literals stream**: - raw literal bytes if `enc_lit=0`, or - RLE tokenized if `enc_lit=1`, or - Huffman-coded if `enc_lit=2` (see [§ 5.3.1 Huffman literal section](#531-huffman-literal-section)). - **Tokens stream**: - one byte per sequence: `(LL << 4) | ML`. - `LL` and `ML` are 4-bit fields. - **Offsets stream**: - `n_sequences × 1` byte if `enc_off=1`, else `n_sequences × 2` bytes LE. - Values are **biased**: stored value = `actual_offset - 1`. Decoder adds `+ 1`. - This makes `offset == 0` impossible by construction (minimum decoded offset = 1). - **Extras stream**: - Prefix-varint overflow values for token saturations: - if `LL == 15`, read varint and add to LL - if `ML == 15`, read varint and add to ML - actual match length is `ML + 5` (minimum match = 5). ### 5.3.1 Huffman literal section Selected by the encoder only at compression level ≥ 6, only when at least `ZXC_HUF_MIN_LITERALS = 1024` literals are present, and only when the Huffman payload is at least ~3 % smaller than the corresponding RAW or RLE encoding of the same literals. Any block where the heuristic does not pick HUFFMAN keeps `enc_lit ∈ {0, 1}`. The Huffman literal section payload is structured as follows: ```text Offset Size Field 0x00 128 Code-length header 256 × 4-bit code lengths, packed two-per-byte (low nibble first). code_len[i] ∈ [0, 8] (0 means symbol absent). 0x80 6 Sub-stream sizes s1, s2, s3 as little-endian u16 (size of streams 0, 1, 2 in bytes). The size of stream 3 is implied: s4 = total_payload_size - 134 - s1 - s2 - s3. 0x86 var Stream 0 bit-stream (s1 bytes, LSB-first) var Stream 1 bit-stream (s2 bytes) var Stream 2 bit-stream (s3 bytes) var Stream 3 bit-stream (s4 bytes) ``` Codes are canonical, length-limited at `L = 8`, emitted **LSB-first**. The `n_literals` value from the GLO header is split into 4 contiguous regions of size `Q = ceil(n_literals / 4)` (the last region may be shorter), each encoded into its own bit-stream so that 4 decoders can run in parallel. The decoder reconstructs the canonical code table from the 128-byte length header, validates the Kraft equality, and decodes each sub-stream into its output region. See [WHITEPAPER §5.8](WHITEPAPER.md) for the multi-symbol 2048-entry lookup table strategy used on the decode hot path. Decoder validation requirements: - Every code length must satisfy `code_len[i] ≤ 8`. - At least one symbol must be present (`code_len[i] != 0` for some `i`). - The Kraft sum `Σ 2^(8 − code_len[i])` over present symbols must equal `2^8`, except for the single-present-symbol degenerate case where exactly one symbol has `code_len = 1` and the Kraft sum is `2^7`. - A failure on any of the above results in `ZXC_ERROR_CORRUPT_DATA`. --- ## 5.4 GHI block (`type=3`) High-throughput LZ format with packed 32-bit sequences. ### GHI payload layout ```text +-------------------------------+ | GHI Header (16 bytes) | +-------------------------------+ | 3 Section Descriptors (24B) | +-------------------------------+ | Literals stream | +-------------------------------+ | Sequences stream (N * 4B) | +-------------------------------+ | Extras stream | +-------------------------------+ ``` ### GHI Header (16 bytes) Same binary layout as GLO header: - `n_sequences`, `n_literals`, `enc_lit`, `enc_litlen`, `enc_mlen`, `enc_off`, reserved. In practice for GHI: - `enc_lit = 0` (raw literals) - `enc_off` is metadata (sequence words always store 16-bit offsets) ### GHI section descriptors (3 × 8 bytes) Section order: 1. Literals 2. Sequences 3. Extras Each descriptor uses the same packed size encoding as GLO (`u64`: comp32|raw32). ### GHI sequence word format (32 bits) ```text Bits 31..24 : LL (literal length, 8 bits) Bits 23..16 : ML (match length minus 5, 8 bits) Bits 15..0 : Offset - 1 (16 bits, biased; decode: stored + 1) ``` Memory order (little-endian word): ```text byte0 = offset low byte1 = offset high byte2 = ML byte3 = LL ``` Overflow rules: - if `LL == 255`, read varint from Extras and add it to LL. - if `ML == 255`, read varint, then add minimum match (`+5`). - otherwise decoded match length is `ML + 5`. --- ## 5.5 EOF block (`type=255`) EOF marks end of block stream. Constraints: - block header is present (8 bytes) - `comp_size` **must be 0** - no payload - no per-block trailing checksum Immediately after EOF block header comes the Optional SEK block, followed by the 12-byte file footer. --- ## 5.6 SEK block (`type=254`) The **Seek Table** block is an optional block appended between the EOF block and the File Footer. It provides `O(1)` random-access capabilities by recording the compressed size of every block in the archive. Decompressed sizes and block indices are derived from the file header's `block_size` (all blocks are `block_size` except the last, which may be smaller). **Layout of a SEK Block**: ```text Offset Size Field 0x00 8 Block Header (type=254, comp_size=N*4) 0x08 4 Block 0 Compressed Size (u32 LE) 0x0C 4 Block 1 Compressed Size (u32 LE) ... ... ... 8 + (N-1)*4 4 Block N-1 Compressed Size (u32 LE) ``` **Backward Detection Strategy**: 1. Read the **File Header** (first 16 bytes) -> extract `block_size`. 2. Read the **File Footer** (last 12 bytes) -> extract `total_decompressed_size`. 3. Derive `num_blocks = ceil(total_decompressed_size / block_size)`. 4. Calculate `seek_block_size = 8 + (N × 4)`. 5. Seek backward by `seek_block_size` bytes from the start of the footer to read the Block Header. 6. Validate `block_type == 254 (SEK)` and `comp_size == N × 4`. --- ## 6. Prefix Varint (Extras stream) ZXC extras use a prefix-length varint. Length is encoded in unary form in the high bits of first byte: - `0xxxxxxx` -> 1 byte total - `10xxxxxx` -> 2 bytes total - `110xxxxx` -> 3 bytes total - `1110xxxx` -> 4 bytes total - `11110xxx` -> 5 bytes total Payload bits from following bytes are concatenated little-endian style (low bits first). Used by GLO/GHI to carry LL/ML overflows beyond token/sequence inline limits. --- ## 7. Checksums and Integrity ## 7.1 Header checksums - File header: 16-bit (`zxc_hash16`). - Block header: 8-bit (`zxc_hash8`). These protect metadata/navigation fields. ## 7.2 Per-block checksum (optional) When file header has `HAS_CHECKSUM=1`: - each data block appends a 4-byte checksum after payload. - checksum input is **compressed payload bytes only** (not block header). - algorithm id currently `0` (RapidHash folded to 32-bit). ## 7.3 Global stream hash A rolling global hash is maintained from per-block checksums in stream order: ```text global = 0 for each data block checksum b: global = ((global << 1) | (global >> 31)) XOR b ``` This value is stored in the file footer (or zeroed when checksum mode is disabled). --- ## 8. File Footer (12 bytes) Footer is mandatory and placed immediately after EOF block header. ```text Offset Size Field 0x00 8 original_source_size (u64) 0x08 4 global_hash (u32) ``` - **original_source_size**: full uncompressed size of the file. - **global_hash**: - valid when checksum mode is active; - set to zero when checksum mode is disabled. --- ## 9. Decoder Validation Checklist (Practical) 1. Validate file header magic/version/CRC16. 2. Parse blocks sequentially: - validate block header CRC8, - check block bounds using `comp_size`, - if enabled, verify trailing block checksum. 3. Decode payload according to block type. 4. On EOF: - require `comp_size == 0`, - read footer, - compare footer `original_source_size` with produced output size, - if enabled, compare footer `global_hash` with recomputed rolling hash. --- ## 10. Versioning Policy ### 10.1 Format version field The format version is a single byte at offset `0x04` of the file header. A conforming decoder **MUST** reject any file whose version it does not support. ### 10.2 Version bump criteria | Change class | Version action | Example | |---|---|---| | New block type added | **No bump** (forward-compatible) | Adding a hypothetical `GLR` block type | | New flag bit defined | **No bump** (forward-compatible) | Using a reserved flag bit | | Existing block encoding changed | **Major bump** | Changing GLO token layout | | Header/footer layout changed | **Major bump** | Resizing the file header | | Checksum algorithm changed | **Major bump** | Replacing RapidHash with Komihash | ### 10.3 Compatibility rules - **Backward compatibility**: a decoder supporting version *N* **MUST** decode all files produced by encoders of version *N*. It **MAY** also accept earlier versions. - **Forward compatibility**: a decoder encountering an **unknown block type** (not RAW, GLO, NUM, GHI, or EOF) **SHOULD** skip it using `comp_size` to advance past its payload (and optional checksum), rather than rejecting the file outright. This allows older decoders to partially process files from newer encoders that introduce additive block types. - **Reserved fields**: all reserved bytes and flag bits **MUST** be written as zero by encoders. Decoders **MUST** ignore reserved fields (not reject non-zero values), unless a future version assigns them meaning. ### 10.4 Minimum conforming decoder A minimal conforming decoder for version 5 **MUST** support: - File header parsing and CRC16 validation. - **RAW** blocks (type 0) - passthrough copy. - **GLO** blocks (type 1) - full LZ decode with extras varint. - **GHI** blocks (type 3) - full LZ decode with extras varint. - **EOF** block (type 255) - stream termination. - File footer validation (source size check). Support for **NUM** (type 2) and checksum verification is **RECOMMENDED** but not strictly required for a minimal implementation. --- ## 11. Error Handling ### 11.1 Error classes Decoders **MUST** detect and handle the following error conditions. The recommended behavior for each class is specified below. | Error | Detection point | Required behavior | |---|---|---| | **Bad magic** | File header, offset 0x00 | Reject immediately. Not a ZXC file. | | **Unsupported version** | File header, offset 0x04 | Reject immediately. Version not supported. | | **Header CRC16 mismatch** | File header, offset 0x0E | Reject. Header is corrupt or truncated. | | **Invalid chunk size code** | File header, offset 0x05 | Reject. Code outside valid range `[12..21]` and not legacy `64`. | | **Block header CRC8 mismatch** | Block header, offset 0x07 | Reject block. Stream is corrupt. | | **Unknown block type** | Block header, offset 0x00 | Skip block using `comp_size` (see §10.3), or reject. | | **Block payload truncated** | During `fread` of `comp_size` bytes | Reject. Unexpected end of stream. | | **Block checksum mismatch** | Trailing 4-byte checksum | Reject block. Payload is corrupt. | | **EOF block with non-zero comp_size** | EOF block header | Reject. Malformed EOF marker. | | **Footer source size mismatch** | File footer, offset 0x00 | Reject. Output size does not match declared original size. | | **Footer global hash mismatch** | File footer, offset 0x08 | Reject (if checksum mode active). Integrity failure. | | **Decompressed output exceeds chunk size** | During LZ decode | Reject. Corrupt or malicious payload. | | **Match offset out of bounds** | During LZ copy | Reject. Offset references data before output start. | | **Varint exceeds maximum length** | Extras stream | Reject. Overflow or corrupt extras data. | ### 11.2 Severity levels - **Fatal**: the decoder **MUST** stop processing and report an error. All errors in the table above are fatal by default. - **Warning**: not currently defined. Future versions may introduce non-fatal conditions (e.g. unknown flag bits set in reserved positions). ### 11.3 Partial output When a fatal error occurs mid-stream, the decoder **SHOULD**: 1. Stop producing output immediately. 2. Report the specific error condition (see `zxc_error_name` in the reference implementation). 3. Not return partially decompressed data as a valid result. Buffer-mode decoders **MUST** return a negative error code. Stream-mode decoders **MUST** signal the error and cease writing to the output. ### 11.4 Decoder hardening recommendations For decoders processing untrusted input (e.g. network data, user uploads): - Validate **all** header checksums before processing payloads. - Enforce maximum allocation limits based on `comp_size` and chunk size code. - Reject files where `comp_size` exceeds `zxc_compress_bound(chunk_size)`. - Use bounded memory copies - never trust decoded lengths without cross-checking against output buffer capacity. --- ## 12. Summary of Useful Fixed Sizes - File header: **16** bytes - Block header: **8** bytes - Block checksum (optional): **4** bytes - NUM header: **16** bytes - GLO header: **16** bytes - GHI header: **16** bytes - Section descriptor: **8** bytes - GLO descriptors total: **32** bytes - GHI descriptors total: **24** bytes - File footer: **12** bytes --- ## 13. Worked Example (Real Hexdump) This example was produced with the CLI from a 10-byte input (`Hello ZXC\n`) using: ```bash zxc -z -C -1 sample.txt ``` Generated archive size: **58 bytes**. ### 13.1 Full hexdump ```text 00000000: F5 2E B0 9C 05 13 80 00 00 00 00 00 00 00 B8 90 00000010: 00 00 00 0A 00 00 00 69 48 65 6C 6C 6F 20 5A 58 00000020: 43 0A 90 BB A1 75 FF 00 00 00 00 00 00 02 0A 00 00000030: 00 00 00 00 00 00 90 BB A1 75 ``` ### 13.2 Byte-level decoding #### A) File Header (offset `0x00`, 16 bytes) ```text F5 2E B0 9C | 05 | 13 | 80 | 00 00 00 00 00 00 00 | B8 90 ``` - `F5 2E B0 9C` -> magic word (LE) = `0x9CB02EF5`. - `05` -> format version 5. - `13` -> chunk-size code 19 (exponent encoding: `2^19 = 524288` bytes, i.e. 512 KiB, the default). - `80` -> checksum enabled (`HAS_CHECKSUM=1`, algo id 0). - next 7 bytes are reserved zeros. - `B8 90` -> header CRC16. #### B) Data Block #0 (RAW) Block header at offset `0x10`: ```text 00 | 00 | 00 | 0A 00 00 00 | 69 ``` - type `00` = RAW. - flags `00`, reserved `00`. - `comp_size = 0x0000000A = 10` bytes. - header CRC8 = `0x69`. Payload at `0x18..0x21` (10 bytes): ```text 48 65 6C 6C 6F 20 5A 58 43 0A ``` ASCII: `Hello ZXC\n`. Trailing block checksum at `0x22..0x25`: ```text 90 BB A1 75 ``` LE value: `0x75A1BB90`. #### C) EOF Block (offset `0x26`, 8 bytes) ```text FF | 00 | 00 | 00 00 00 00 | 02 ``` - type `FF` = EOF. - `comp_size = 0` (mandatory). - header CRC8 = `0x02`. #### D) File Footer (offset `0x2E`, 12 bytes) ```text 0A 00 00 00 00 00 00 00 | 90 BB A1 75 ``` - original source size = `10` bytes. - global hash = `0x75A1BB90`. Since there is exactly one data block, the global hash equals that block checksum: ```text global0 = 0 global1 = rotl1(global0) XOR block_crc = block_crc ``` ### 13.3 Structural view with absolute offsets ```text 0x00..0x0F File Header (16) 0x10..0x17 RAW Block Header (8) 0x18..0x21 RAW Payload (10) 0x22..0x25 RAW Block Checksum (4) 0x26..0x2D EOF Block Header (8) 0x2E..0x39 File Footer (12) ``` ### 13.4 Seekable Variant (with Seek Table) Same 10-byte input (`Hello ZXC\n`), compressed with seekable mode enabled: ```bash zxc -z -C -1 -S sample.txt ``` Generated archive size: **70 bytes** (12 bytes larger than the non-seekable variant). #### Full hexdump ```text 00000000: F5 2E B0 9C 05 13 80 00 00 00 00 00 00 00 B8 90 00000010: 00 00 00 0A 00 00 00 69 48 65 6C 6C 6F 20 5A 58 00000020: 43 0A 90 BB A1 75 FF 00 00 00 00 00 00 02 FE 00 00000030: 00 04 00 00 00 D2 16 00 00 00 0A 00 00 00 00 00 00000040: 00 00 90 BB A1 75 ``` #### Byte-level decoding **A) File Header** (offset `0x00`, 16 bytes) - identical to non-seekable. **B) Data Block #0 (RAW)** (offset `0x10`, 22 bytes) - identical to non-seekable. **C) EOF Block** (offset `0x26`, 8 bytes) - identical to non-seekable. **D) SEK Block** (offset `0x2E`, 12 bytes) Block header at `0x2E`: ```text FE | 00 | 00 | 04 00 00 00 | D2 ``` - `FE` -> type 254 = SEK (Seek Table). - flags `00`, reserved `00`. - `comp_size = 0x00000004 = 4` bytes (one entry x 4 bytes/entry). - header CRC8 = `0xD2`. Seek table entry at `0x36`: ```text 16 00 00 00 ``` - Entry #0: compressed block size = `0x00000016 = 22` bytes. This is the total size of data block #0 including its header (8) + payload (10) + checksum (4) = 22. ✓ **E) File Footer** (offset `0x3A`, 12 bytes) ```text 0A 00 00 00 00 00 00 00 | 90 BB A1 75 ``` - original source size = `10` bytes. - global hash = `0x75A1BB90`. #### Structural view with absolute offsets ```text 0x00..0x0F File Header (16) 0x10..0x17 RAW Block Header (8) 0x18..0x21 RAW Payload (10) 0x22..0x25 RAW Block Checksum (4) 0x26..0x2D EOF Block Header (8) 0x2E..0x35 SEK Block Header (8) <- seek table 0x36..0x39 SEK Entry #0 (4) <- comp_size of block #0 0x3A..0x45 File Footer (12) ``` > **Compatibility note**: The SEK block is inserted between the EOF block and the file footer. The footer always remains the **last 12 bytes of the file**, so decoders that locate the footer from the end of the file (e.g. `src + src_size - 12` for buffer APIs, or `fseek(END - 12)` for file APIs) work unchanged with seekable archives. However, **streaming decoders** that read the footer sequentially immediately after the EOF block must be updated to detect and skip the SEK block. In practice, all ZXC decoders since v0.9.0 handle both seekable and non-seekable archives transparently. zxc-0.11.0/docs/WHITEPAPER.md000066400000000000000000001645251520102567100153330ustar00rootroot00000000000000# ZXC: High-Performance Asymmetric Lossless Compression **Version**: 0.11.0 **Date**: May 2026 **Author**: Bertrand Lebonnois --- ## 1. Executive Summary In modern software delivery pipelines-specifically **Mobile Gaming**, **Embedded Systems**, and **FOTA (Firmware Over-The-Air)**-data is typically generated on high-performance x86 workstations but consumed on energy-constrained ARM devices. Standard industry codecs like LZ4 offer excellent performance but fail to exploit the "Write-Once, Read-Many" (WORM) nature of these pipelines. **ZXC** is a lossless codec designed to bridge this gap. By utilizing an **asymmetric compression model**, ZXC achieves a **>40% increase in decompression speed on ARM** compared to LZ4, while simultaneously reducing storage footprints. On x86 development architecture, ZXC maintains competitive throughput, ensuring no disruption to build pipelines. ## 2. The Efficiency Gap The industry standard, LZ4, prioritizes symmetric speed (fast compression and fast decompression). While ideal for real-time logs or RAM swapping, this symmetry is useless for asset distribution. * **Wasted Cycles**: CPU cycles saved during the single compression event (on a build server) do not benefit the millions of end-users decoding the data. * **The Battery Tax**: On mobile devices, slower decompression keeps the CPU active longer, draining battery and generating heat. ## 3. The ZXC Solution ZXC utilizes a computationally intensive encoder to generate a bitstream specifically structured to **maximize decompression throughput**. By performing heavy analysis upfront, the encoder produces a layout optimized for the instruction pipelining and branch prediction capabilities of modern CPUs, particularly ARMv8, effectively offloading complexity from the decoder to the encoder. ### 3.1 Asymmetric Pipeline ZXC employs a Producer-Consumer architecture to decouple I/O operations from CPU-intensive tasks. This allows for parallel processing where input reading, compression/decompression, and output writing occur simultaneously, effectively hiding I/O latency. ### 3.2 Modular Architecture The ZXC file format is inherently modular. **Each block is independent and can be encoded and decoded using the algorithm best suited** for that specific data type. This flexibility allows the format to evolve and incorporate new compression strategies without breaking backward compatibility. ## 4. Core Algorithms ZXC utilizes a hybrid approach combining LZ77 (Lempel-Ziv) dictionary matching with advanced entropy coding and specialized data transforms. ### 4.1 LZ77 Engine The heart of ZXC is a heavily optimized LZ77 engine that adapts its behavior based on the requested compression level: * **Hash Chain & Collision Resolution**: Uses a fast hash table with chaining to find matches in the history window (configurable sliding window, power-of-2 from 4 KB to 2 MB, default 512 KB). * **Lazy Matching**: Implements a "lookahead" strategy to find better matches at the cost of slight encoding speed, significantly improving decompression density. ### 4.2 Specialized SIMD Acceleration & Hardware Hashing ZXC leverages modern instruction sets to maximize throughput on both ARM and x86 architectures. * **ARM NEON Optimization**: Extensive usage of vld1q_u8 (vector load) and vceqq_u8 (parallel comparison) allows scanning data at wire speed, while vminvq_u8 provides fast rejection of non-matches. * **x86 Vectorization**: Maintains high performance on Intel/AMD platforms via dedicated AVX2 and AVX512 paths (falling back to SSE4.1 on older hardware), ensuring parity with ARM throughput. * **High-Speed Integrity**: Block validation relies on **rapidhash**, a modern non-cryptographic hash algorithm that fully exploits hardware acceleration to verify data integrity without bottlenecking the decompression pipeline. ### 4.3 Entropy Coding & Bitpacking * **RLE (Run-Length Encoding)**: Automatically detects runs of identical bytes. * **Prefix Varint Encoding**: Variable-length integer encoding (similar to LEB128 but prefix-based) for overflow values. * **Canonical Huffman (Literals, level ≥ 6)**: Length-limited (`L = 8`) canonical Huffman code over the literal byte distribution, split into 4 LSB-first interleaved bit-streams. Decoded via a cache-line-aligned 2048-entry lookup table (11-bit window) that returns **1 or 2 symbols per access** depending on whether the next two codes fit the window. On typical literal distributions ~50–70 % of lookups yield 2 symbols, pushing effective throughput above one symbol per memory access. * **Bit-Packing**: Compressed sequences are packed into dedicated streams using minimal bit widths. #### Multi-Symbol Huffman LUT The Huffman decoder is the hot path for high-compression levels, so ZXC trades a larger table for fewer lookups. The classical trade-off is between table size (cache footprint) and the number of symbols decoded per memory access. ZXC's design: * **Length limit `L = 8`**: With a maximum codeword length of 8 bits, the longest *pair* of codes never exceeds 16 bits, which keeps the multi-symbol lookup tractable. * **11-bit window**: The LUT is indexed by 11 bits read ahead from the stream. Each entry stores either a single symbol (when the first code is ≥ 4 bits and the cumulative length of the *first two codes* would exceed 11 bits) or a pair of symbols (when both fit). This single branch (fits-in-window?) is resolved by a precomputed flag in the LUT entry, no per-lookup re-decoding. * **2048-entry, cache-aligned**: 2048 × 4-byte entries = 8 KB, aligned on a cache line. Easily fits L1d on every target CPU. * **4 parallel bit-streams**: Literals are interleaved across 4 LSB-first streams so each decoder consumes independent bits, breaking the serial dependency that limits classical Huffman to one symbol per cycle. * **Scalar tail**: A small per-stream scalar epilogue (≤ 9 symbols) handles the trailing bytes safely without speculative writes past the destination buffer. Selection is conservative: `enc_lit = 2` (Huffman) is chosen only if the Huffman section is at least ~3 % smaller than the RAW or RLE baseline, avoiding setup overhead on near-uniform literal distributions where the entropy savings would not justify the decoder cost. #### Prefix Varint Format ZXC uses a **Prefix Varint** encoding for overflow values. Unlike standard VByte (which uses a continuation bit in every byte), Prefix Varint encodes the total length of the integer in the **unary prefix of the first byte**. This allows the decoder to determine the sequence length immediately, enabling branchless or highly predictable decoding without serial dependencies. **Encoding Scheme:** | Prefix (Binary) | Total Bytes | Data Bits (1st Byte) | Total Data Bits | Range (Value < X) | |-----------------|-------------|----------------------|-----------------|-------------------| | `0xxxxxxx` | 1 | 7 | 7 | 128 | | `10xxxxxx` | 2 | 6 | 14 (6+8) | 16,384 | | `110xxxxx` | 3 | 5 | 21 (5+8+8) | 2,097,152 | | `1110xxxx` | 4 | 4 | 28 (4+8+8+8) | 268,435,456 | | `11110xxx` | 5 | 3 | 35 (3+8+8+8+8) | 34,359,738,368 | **Example**: Encoding value `300` (binary: `100101100`): ```text Value 300 > 127 and < 16383 -> Uses 2-byte format (Prefix '10'). Step 1: Low 6 bits 300 & 0x3F = 44 (0x2C, binary 101100) Byte 1 = Prefix '10' | 101100 = 10101100 (0xAC) Step 2: Remaining high bits 300 >> 6 = 4 (0x04, binary 00000100) Byte 2 = 0x04 Result: 0xAC 0x04 Decoding Verification: Byte 1 (0xAC) & 0x3F = 44 Byte 2 (0x04) << 6 = 256 Total = 256 + 44 = 300 ``` ## 5. File Format Specification The ZXC file format is block-based, robust, and designed for parallel processing. ### 5.1 Global Structure (File Header) The file begins with a **16-byte** header that identifies the format and specifies decompression parameters. **FILE Header (16 bytes):** ``` Offset: 0 4 5 6 7 14 16 +---------------+-------+-------+-------+-----------------------+-------+ | Magic Word | Ver | Chunk | Flags | Reserved | CRC | | (4 bytes) | (1B) | (1B) | (1B) | (7 bytes, must be 0) | (2B) | +---------------+-------+-------+-------+-----------------------+-------+ ``` * **Magic Word (4 bytes)**: `0x9 0xCB 0x02E 0xF5`. * **Version (1 byte)**: Current version is `5`. * **Chunk Size Code (1 byte)**: Defines the processing block size using **exponent encoding**: - If the value is in `[12, 21]`: block size = `2^value` bytes (4 KB to 2 MB). - `12` = 4 KB, `13` = 8 KB, `14` = 16 KB, `15` = 32 KB, `16` = 64 KB, `17` = 128 KB, `18` = 256 KB, `19` = 512 KB (default), `20` = 1 MB, `21` = 2 MB. - Legacy value `64` is accepted for backward compatibility (maps to 256 KB). - Block sizes must be powers of 2. * **Flags (1 byte)**: Global configuration flags. - **Bit 7 (MSB)**: `HAS_CHECKSUM`. If `1`, checksums are enabled for the stream. Every block will carry a trailing 4-byte checksum, and the footer will contain a global checksum. If `0`, no checksums are present. - **Bits 4-6**: Reserved. - **Bits 0-3**: Checksum Algorithm ID (e.g., `0` = RapidHash). * **Reserved (7 bytes)**: Reserved for future use (must be 0). * **CRC (2 bytes)**: 16-bit Header Checksum. Calculated on the 16-byte header (with CRC bytes set to 0) using `zxc_hash16`. ### 5.2 Block Header Structure Each data block consists of an **8-byte** generic header that precedes the specific payload. This header allows the decoder to navigate the stream and identify the processing method required for the next chunk of data. **BLOCK Header (8 bytes):** ``` Offset: 0 1 2 3 7 8 +-------+-------+-------+-----------------------+-------+ | Type | Flags | Rsrvd | Comp Size | CRC | | (1B) | (1B) | (1B) | (4 bytes) | (1B) | +-------+-------+-------+-----------------------+-------+ Block Layout: [ Header (8B) ] + [ Compressed Payload (Comp Size bytes) ] + [ Optional Checksum (4B) ] ``` **Note**: The Checksum (if enabled in File Header) is **4 bytes** (32-bit), is always located **at the end** of the compressed data, and is calculated **on the compressed payload**. * **Type**: Block encoding type (0=RAW, 1=GLO, 2=NUM, 3=GHI, 255=EOF). * **Flags**: Not used for now. * **Rsrvd**: Reserved for future use (must be 0). * **Comp Size**: Compressed payload size (excluding header and optional checksum). * **CRC**: 1-byte Header Checksum (located at the end of the header). Calculated on the 8-byte header (with CRC byte set to 0) using `zxc_hash8`. > **Note**: The decompressed size is not stored in the block header. It is derived from internal Section Descriptors within the compressed payload (for GLO/GHI blocks), from the NUM header (for NUM blocks), or equals `Comp Size` (for RAW blocks). > **Note**: While the format is designed for threaded execution, a single-threaded API is also available for constrained environments or simple integration cases. ### 5.3 Specific Header: NUM (Numeric) (Present immediately after the Block Header) **NUM Header (16 bytes):** ``` Offset: 0 8 10 16 +-------------------------------+-------+-------------------------+ | N Values | Frame | Reserved | | (8 bytes) | (2B) | (6 bytes) | +-------------------------------+-------+-------------------------+ ``` * **N Values**: Total count of integers encoded in the block. * **Frame**: Processing window size (currently always 128). * **Reserved**: Padding for alignment. ### 5.4 Specific Header: GLO (Generic Low) (Present immediately after the Block Header) **GLO Header (16 bytes):** ``` Offset: 0 4 8 9 10 11 12 16 +---------------+---------------+---+---+---+---+---------------+ | N Sequences | N Literals |Lit|LL |ML |Off| Reserved | | (4 bytes) | (4 bytes) |Enc|Enc|Enc|Enc| (4 bytes) | +---------------+---------------+---+---+---+---+---------------+ ``` * **N Sequences**: Total count of LZ sequences in the block. * **N Literals**: Total count of literal bytes. * **Encoding Types** - `Lit Enc`: Literal stream encoding (0=RAW, 1=RLE, 2=HUFFMAN). **Currently used.** - `LL Enc`: Literal lengths encoding. **Reserved for future use** (lengths are packed in tokens). - `ML Enc`: Match lengths encoding. **Reserved for future use** (lengths are packed in tokens). - `Off Enc`: Offset encoding mode. **Currently used** - `0` = 16-bit offsets (2 bytes each, max distance 65535) - `1` = 8-bit offsets (1 byte each, max distance 255) * **Reserved**: Padding for alignment. **Section Descriptors (4 × 8 bytes = 32 bytes total):** Each descriptor stores sizes as a packed 64-bit value: ``` Single Descriptor (8 bytes): +-----------------------------------+-----------------------------------+ | Compressed Size (4 bytes) | Raw Size (4 bytes) | | (low 32 bits) | (high 32 bits) | +-----------------------------------+-----------------------------------+ Full Layout (32 bytes): Offset: 0 8 16 24 32 +---------------+---------------+---------------+---------------+ | Literals Desc | Tokens Desc | Offsets Desc | Extras Desc | | (8 bytes) | (8 bytes) | (8 bytes) | (8 bytes) | +---------------+---------------+---------------+---------------+ ``` **Section Contents:** | # | Section | Description | |---|-------------|-------------------------------------------------------| | 0 | **Literals**| Raw bytes to copy, or RLE-compressed if `enc_lit=1` | | 1 | **Tokens** | Packed bytes: `(LiteralLen << 4) \| MatchLen` | | 2 | **Offsets** | Match distances: 8-bit if `enc_off=1`, else 16-bit LE | | 3 | **Extras** | Prefix Varint overflow values when LitLen or MatchLen ≥ 15 | **Data Flow Example:** ``` GLO Block Data Layout: +------------------------------------------------------------------------+ | Literals Stream | Tokens Stream | Offsets Stream | Extras Stream | | (desc[0] bytes) | (desc[1] bytes)| (desc[2] bytes)| (desc[3] bytes) | +------------------------------------------------------------------------+ ↓ ↓ ↓ ↓ Raw bytes Token parsing Match lookup Length overflow ``` **Why Comp Size and Raw Size?** Each descriptor stores both a compressed and raw size to support secondary encoding of streams: | Section | Comp Size | Raw Size | Different? | |-------------|----------------------|---------------------|----------------------| | **Literals**| RLE size (if used) | Original byte count | Yes, if RLE enabled | | **Tokens** | Stream size | Stream size | No | | **Offsets** | N×1 or N×2 bytes | N×1 or N×2 bytes | No (size depends on `enc_off`) | | **Extras** | Prefix Varint stream size | Prefix Varint stream size | No | Currently, the **Literals** section uses different sizes when RLE compression is applied (`enc_lit=1`). The **Offsets** section size depends on `enc_off`: N sequences × 1 byte (if `enc_off=1`) or N sequences × 2 bytes (if `enc_off=0`). > **Design Note**: This format is designed for future extensibility. The dual-size architecture allows adding entropy coding (FSE/ANS) or bitpacking to any stream without breaking backward compatibility. ### 5.5 Specific Header: GHI (Generic High) (Present immediately after the Block Header) The **GHI** (Generic High-Velocity) block format is optimized for maximum decompression speed. It uses a **packed 32-bit sequence** format that allows 4-byte aligned reads, reducing memory access latency and enabling efficient SIMD processing. **GHI Header (16 bytes):** ``` Offset: 0 4 8 9 10 11 12 16 +---------------+---------------+---+---+---+---+---------------+ | N Sequences | N Literals |Lit|LL |ML |Off| Reserved | | (4 bytes) | (4 bytes) |Enc|Enc|Enc|Enc| (4 bytes) | +---------------+---------------+---+---+---+---+---------------+ ``` * **N Sequences**: Total count of LZ sequences in the block. * **N Literals**: Total count of literal bytes. * **Encoding Types** - `Lit Enc`: Literal stream encoding (0=RAW). - `LL Enc`: Reserved for future use. - `ML Enc`: Reserved for future use. - `Off Enc`: Offset encoding mode: - `0` = 16-bit offsets (max distance 65535) - `1` = 8-bit offsets (max distance 255, enables smaller sequence packing) * **Reserved**: Padding for alignment. **Section Descriptors (3 × 8 bytes = 24 bytes total):** ``` Full Layout (24 bytes): Offset: 0 8 16 24 +---------------+---------------+---------------+ | Literals Desc | Sequences Desc| Extras Desc | | (8 bytes) | (8 bytes) | (8 bytes) | +---------------+---------------+---------------+ ``` **Section Contents:** | # | Section | Description | |---|---------------|-------------------------------------------------------| | 0 | **Literals** | Raw bytes to copy | | 1 | **Sequences** | Packed 32-bit sequences (see format below) | | 2 | **Extras** | Prefix Varint overflow values when LitLen or MatchLen ≥ 255 | **Packed Sequence Format (32 bits):** Unlike GLO which uses separate token and offset streams, GHI packs all sequence data into a single 32-bit word for cache-friendly sequential access: ``` 32-bit Sequence Word (Little Endian): +--------+--------+------------------+ | LL | ML | Offset | | 8 bits | 8 bits | 16 bits | +--------+--------+------------------+ [31:24] [23:16] [15:0] Byte Layout in Memory: Offset: 0 1 2 3 +--------+--------+--------+--------+ | Off Lo | Off Hi | ML | LL | +--------+--------+--------+--------+ ``` * **LL (Literal Length)**: 8 bits (0-254, value 255 triggers Prefix Varint overflow) * **ML (Match Length - 5)**: 8 bits (actual length = ML + 5, range 5-259, value 255 triggers Prefix Varint overflow) * **Offset**: 16 bits (match distance, 1-65535) **Data Flow Example:** ``` GHI Block Data Layout: +------------------------------------------------------------+ | Literals Stream | Sequences Stream | Extras Stream | | (desc[0] bytes) | (desc[1] bytes = N×4) | (desc[2] bytes) | +------------------------------------------------------------+ ↓ ↓ ↓ Raw bytes 32-bit seq read Length overflow ``` **Key Differences: GLO vs GHI** | Feature | GLO (Global) | GHI (High-Velocity) | |--------------------|---------------------------------|----------------------------------| | **Sections** | 4 (Lit, Tokens, Offsets, Extras)| 3 (Lit, Sequences, Extras) | | **Sequence Format**| 1-byte token + separate offset | Packed 32-bit word | | **LL/ML Bits** | 4 bits each (overflow at 15) | 8 bits each (overflow at 255) | | **Memory Access** | Multiple stream pointers | Single aligned 4-byte reads | | **Decoder Speed** | Fast | Fastest (optimized for ARM/x86) | | **RLE Support** | Yes (literals) | No | | **Huffman Literals** | Yes (level ≥ 6, ≥ 1024 lits) | No | | **Parser** | Lazy (≤ L5), Optimal DP (L6) | Lazy | | **Best For** | General data, good compression | Maximum decode throughput | > **Design Rationale**: The 32-bit packed format eliminates pointer chasing between token and offset streams. By reading a single aligned word per sequence, the decoder achieves better cache utilization and enables aggressive loop unrolling (4x) for maximum throughput on modern CPUs. ### 5.6 Specific Header: EOF (End of File) (Block Type 255) The **EOF** block marks the end of the ZXC stream. It ensures that the decompressor knows exactly when to stop processing, allowing for robust stream termination even when file size metadata is unavailable or when concatenating streams. * **Structure**: Standard 8-byte Block Header. * **Flags**: * **Bit 7 (0x80)**: `has_checksum`. If set, implies the **Global Stream Checksum** in the footer is valid and should be verified. * **Comp Size**: Unlike other blocks, these **MUST be set to 0**. The decoder enforces strict validation (`Type == EOF` AND `Comp Size == 0`) to prevent processing of malformed termination blocks. * **CRC**: 1-byte Header Checksum (located at the end of the header). Calculated on the 8-byte header (with CRC byte set to 0) using `zxc_hash8`. ### 5.7 File Footer (Present immediately after the EOF Block) A mandatory **12-byte footer** closes the stream, providing total source size information and the global checksum. **Footer Structure (12 bytes):** ``` Offset: 0 8 12 +-------------------------------+---------------+ | Original Source Size | Global Hash | | (8 bytes) | (4 bytes) | +-------------------------------+---------------+ ``` * **Original Source Size** (8 bytes): Total size of the uncompressed data. * **Global Hash** (4 bytes): The **Global Stream Checksum**. Valid only if the EOF block has the `has_checksum` flag set (or the decoder context requires it). * **Algorithm**: `Rotation + XOR`. * For each block with a checksum: `global_hash = (global_hash << 1) | (global_hash >> 31); global_hash ^= block_hash;` ### 5.8 Block Encoding & Processing Algorithms The efficiency of ZXC relies on specialized algorithmic pipelines for each block type. #### Type 1: GLO (Global) This format is used for standard data. It employs a **multi-stage encoding pipeline**: **Encoding Process**: 1. **LZ77 Parsing**: The encoder iterates through the input using a rolling hash to detect matches. * *Hash Chain*: Collisions are resolved via a chain table to find optimal matches in dense data. * *Lazy Matching* (levels 3–5): If a match is found, the encoder checks the next position. If a better match starts there, the current byte is emitted as a literal (deferred matching). * *Price-Based Optimal Parser* (level 6): A forward dynamic-programming pass replaces the lazy parser. `dp[p]` holds the minimum bit-cost to encode `src[0..p)`; transitions consider either emitting a single literal or any sub-length of the longest match found at `p`, using static prices (literal ≈ 9 bits, match ≈ 24 bits + varint extras). Backtracking from `dp[N]` yields the globally optimal token sequence. A long-match guard skips re-search at intra-match positions to keep the parser O(N) on highly repetitive data. 2. **Tokenization**: Matches are split into three components: * *Literal Length*: Number of raw bytes before the match. * *Match Length*: Duration of the repeated pattern. * *Offset*: Distance back to the pattern start. 3. **Stream Separation**: These components are routed to separate buffers: * *Literals Buffer*: Raw bytes. * *Tokens Buffer*: Packed `(LitLen << 4) | MatchLen`. * *Offsets Buffer*: Variable-width distances (8-bit or 16-bit, see below). * *Extras Buffer*: Overflow values for lengths >= 15 (Prefix Varint encoded). * *Offset Mode Selection*: The encoder tracks the maximum offset across all sequences. If all offsets are ≤ 255, the 8-bit mode (`enc_off=1`) is selected, saving 1 byte per sequence compared to 16-bit mode. 4. **RLE Pass**: The literals buffer is scanned for run-length encoding opportunities (runs of identical bytes). If beneficial (>10% gain), it is compressed in place. 5. **Huffman Pass** (level ≥ 6 only, ≥ 1024 literals only): A length-limited canonical Huffman code (`L = 8`) is fitted to the literal byte distribution and the literals are split into 4 LSB-first interleaved bit-streams. The encoding is selected (`enc_lit = 2`) only if it is at least ~3 % smaller than the chosen RAW or RLE baseline. 6. **Final Serialization**: All buffers are concatenated into the payload, preceded by section descriptors. **Decoding Process**: 1. **Deserizalization**: The decoder reads the section descriptors to obtain pointers to the start of each stream (Literals, Tokens, Offsets). 2. **Literal Decompression**: * `enc_lit = 0` (RAW): zero-copy view into the source buffer. * `enc_lit = 1` (RLE): single pass that expands runs and copies literal chunks. * `enc_lit = 2` (HUFFMAN): canonical Huffman section decoded by 4 parallel decoders sharing a cache-line-aligned 2048-entry lookup table (11-bit window). Each lookup returns 1 or 2 symbols depending on whether the cumulative length of the next two codes fits in the 11-bit window — on typical literal distributions ~50–70 % of lookups yield 2 symbols, raising effective throughput well above one symbol per memory access. A small per-stream scalar tail (≤ 9 symbols) handles the trailing bytes safely without speculative writes. 3. **Vertical Execution**: The main loop reads from all three streams simultaneously. 4. **Wild Copy**: * *Literals*: Copied using unaligned 16-byte SIMD loads/stores (`vld1/vst1` on ARM). * *Matches*: Copied using 16-byte stores. Overlapping matches (e.g., repeating pattern "ABC" for 100 bytes) are handled naturally by the CPU's store forwarding or by specific overlapped-copy primitives. * **Safety**: A "Safe Zone" at the end of the buffer forces a switch to a cautious byte-by-byte loop, allowing the main loop to run without bounds checks. #### Type 3: GHI (High-Velocity) This format prioritizes decompression throughput over compression ratio. It uses a **unified sequence stream**: **Encoding Process**: 1. **LZ77 Parsing**: Same as GLO, with aggressive lazy matching and step skipping for optimal matches. 2. **Sequence Packing**: Each match is packed into a 32-bit word: * Bits [31:24]: Literal Length (8 bits) * Bits [23:16]: Match Length - 5 (8 bits) * Bits [15:0]: Offset (16 bits) 3. **Stream Assembly**: Only three streams are generated: * *Literals Buffer*: Raw bytes (no RLE). * *Sequences Buffer*: Packed 32-bit words (4 bytes each). * *Extras Buffer*: Prefix Varint overflow values for lengths >= 255. 4. **Final Serialization**: Streams are concatenated with 3 section descriptors. **Decoding Process**: 1. **Single-Read Loop**: The decoder reads one 32-bit word per sequence, extracting LL, ML, and offset in a single operation. 2. **4x Unrolled Fast Path**: When sufficient buffer margin exists, the decoder processes 4 sequences per iteration: * Pre-reads 4 sequences into registers * Copies literals and matches with 32-byte SIMD operations * Minimal branching for maximum instruction-level parallelism 3. **Offset Validation Threshold**: For the first 256 (8-bit mode) or 65536 (16-bit mode) bytes, offsets are validated against written bytes. After this threshold, all offsets are guaranteed valid. 4. **Wild Copy**: Same 32-byte SIMD copies as GLO, with special handling for overlapping matches (offset < 32). #### Type 2: NUM (Numeric) Triggered when data is detected as a dense array of 32-bit integers. **Encoding Process**: 1. **Vectorized Delta**: Computes `delta[i] = val[i] - val[i-1]` using SIMD integers (AVX2/NEON). 2. **ZigZag Transform**: Maps signed deltas to unsigned space: `(d << 1) ^ (d >> 31)`. 3. **Bit Analysis**: Determines the maximum number of bits `B` needed to represent the deltas in a 128-value frame. 4. **Bit-Packing**: Packs 128 integers into `128 * B` bits. **Decoding Process**: 1. **Bit-Unpacking**: Unpacks bitstreams back into integers. 2. **ZigZag Decode**: Reverses the mapping. 3. **Integration**: Computes the prefix sum (cumulative addition) to restore original values. *Note: ZXC utilizes a 4x unrolled loop here to pipeline the dependency chain.* ### 5.9 Data Integrity Every compressed block can optionally be protected by a **32-bit checksum** to ensure data reliability. #### Post-Compression Verification Unlike traditional codecs that verify the integrity of the original uncompressed data, ZXC calculates checksums on the **compressed** payload. * **Zero-Overhead Decompression**: Verifying uncompressed data requires computing a hash over the output *after* decompression, contending for cache and CPU resources with the decompression logic itself. By checksumming the compressed stream, verification happens *during* the read phase, before the data even enters the decoder. * **Early Failure Detection**: Corruption is detected before attempting to decompress, preventing potential crashes or buffer overruns in the decoder caused by malformed data. * **Reduced Memory Bandwidth**: The checksum is computed over a much smaller dataset (the compressed block), saving significant memory bandwidth. #### Multi-Algorithm Support ZXC supports multiple integrity verification algorithms (though currently standardized on rapidhash). * **Identified Algorithm (0x00: rapidhash)**: The default algorithm. The 64-bit rapidhash result is folded (XORed) into a 32-bit value to minimize storage overhead while maintaining strong collision resistance for block-level integrity. * **Performance First**: By using a modern non-cryptographic hash, ZXC ensures that integrity checks do not bottleneck decompression throughput. #### Credit The default `rapidhash` algorithm is based on wyhash and was developed by Nicolas De Carli. It is designed to fully exploit hardware performance while maintaining top-tier mathematical distribution qualities. ### 5.10 Seekable Archives (Random Access) ZXC supports **O(1)** random-access decompression without decoding the entire stream. This is achieved by appending an optional **Seek Table** (a `SEK` block) at the end of the archive, immediately before the file footer. * **Structure**: The seek table contains an array of 4-byte entries (compressed block size, LE uint32) for every block in the archive. * **Performance**: Reading backward from the file footer instantly locates the seek table. Since blocks have a fixed power-of-2 size, the target block is found by a single division (`block_index = offset / block_size`), with no binary search required. * **Use Cases**: This feature transforms ZXC from a sequential stream into a random-access volume format. ## 6. System Architecture (Threading) ZXC leverages a threaded **Producer-Consumer** model to saturate modern multi-core CPUs. ### 6.1 Asynchronous Compression Pipeline 1. **Block Splitting (Main Thread)**: The input file is read and sliced into fixed-size chunks (configurable, default 512 KB, power of 2 from 4 KB to 2 MB). 2. **Ring Buffer Submission**: Chunks are placed into a lock-free ring buffer. 3. **Parallel Compression (Worker Threads)**: * Workers pull chunks from the queue. * Each worker compresses its chunk independently in its own context (`zxc_cctx_t`). * Output is written to a thread-local buffer. 4. **Reordering & Write (Writer Thread)**: The writer thread ensures chunks are written to disk in the correct original order, regardless of which worker finished first. ### 6.2 Asynchronous Decompression Pipeline 1. **Header Parsing (Main Thread)**: The main thread scans block headers to identify boundaries and payload sizes. 2. **Dispatch**: Compressed payloads are fed into the worker job queue. 3. **Parallel Decoding (Worker Threads)**: * Workers decode chunks into pre-allocated output buffers. * **Fast Path**: If the output buffer has sufficient margin, the decoder uses "wild copies" (16-byte SIMD stores) to bypass bounds checking for maximal speed. 4. **Serialization**: Decompressed blocks are committed to the output stream sequentially. ## 7. Performance Analysis (Benchmarks) **Methodology:** Benchmarks were conducted using `lzbench` (by inikep) with a **block size of 256 KB**, checksums disabled, single-threaded execution, on the standard Silesia Corpus ([silesia.tar](https://github.com/DataCompression/corpus-collection/tree/main/Silesia-Corpus), 202 MB). * **Target 1 (Client):** Apple M2 / macOS 26 (Clang 21) * **Target 2 (Cloud):** Google Axion / Linux (GCC 14) * **Target 3 (Build):** AMD EPYC 9B45 / Linux (GCC 14) * **Target 4 (Production):** AMD EPYC 7763 / Linux (GCC 14) **Figure A**: Pareto Frontier — Decompression Speed vs. Compressed Size (across 4 CPUs) ![Pareto Frontier — Decompression Speed vs Compressed Size](./images/bench-pareto-ratio.svg) ### 7.1 Client ARM64 Summary (Apple Silicon M2) | Compressor | Decompression Speed (Ratio vs LZ4) | Compressed Size (Index LZ4=100) (Lower is Better) | | :--- | :--- | :--- | | **zxc 0.11.0 -1** | **2.62x** | **129.22** | | **zxc 0.11.0 -2** | **2.17x** | **112.64** | | **zxc 0.11.0 -3** | **1.47x** | **96.20** | | **zxc 0.11.0 -4** | **1.40x** | **89.60** | | **zxc 0.11.0 -5** | **1.31x** | **84.59** | | **zxc 0.11.0 -6** | **1.17x** | **76.21** | | lz4 1.10.0 --fast -17 | 1.18x | 130.58 | | lz4 1.10.0 (Ref) | 1.00x | 100.00 | | lz4hc 1.10.0 -9 | 0.95x | 77.20 | | lzav 5.7 -1 | 0.81x | 83.91 | | snappy 1.2.2 | 0.68x | 100.53 | | zstd 1.5.7 --fast --1 | 0.53x | 86.16 | | zstd 1.5.7 -1 | 0.38x | 72.55 | | zlib 1.3.1 -1 | 0.09x | 76.58 | **Decompression Efficiency (Cycles per Byte @ 3.5 GHz)** | Compressor. | Cycles/Byte | Performance vs memcpy (*) | | ----------------------- | ----------- | --------------------- | | memcpy | 0.066 | 1.00x (baseline) | | **zxc 0.11.0 -1** | **0.279** | **4.2x** | | **zxc 0.11.0 -2** | **0.338** | **5.1x** | | **zxc 0.11.0 -3** | **0.497** | **7.5x** | | **zxc 0.11.0 -4** | **0.523** | **7.9x** | | **zxc 0.11.0 -5** | **0.559** | **8.4x** | | **zxc 0.11.0 -6** | **0.623** | **9.4x** | | lz4 1.10.0 | 0.732 | 11.1x | | lz4 1.10.0 --fast -17 | 0.622 | 9.4x | | lz4hc 1.10.0 -9 | 0.773 | 11.7x | | lzav 5.7 -1 | 0.903 | 13.6x | | zstd 1.5.7 -1 | 1.938 | 29.3x | | zstd 1.5.7 --fast --1 | 1.379 | 20.8x | | snappy 1.2.2 | 1.072 | 16.2x | | zlib 1.3.1 -1 | 8.537 | 129x | *Lower is better. Calculated using Apple M2 Performance Core frequency (3.5 GHz). Formula: `Cycles/Byte = 3500 / Decompression Speed (MB/s)`.* **Effective Throughput (Ratio-normalized decode)** | Compressor | Decode (MB/s) | Ratio (%) | Effective (MB/s) | vs LZ4 | | :--- | ---: | ---: | ---: | ---: | | **zxc 0.11.0 -1** | 12 530 | 61.50 | **20 374** | **2.03x** | | **zxc 0.11.0 -2** | 10 360 | 53.61 | **19 324** | **1.92x** | | **zxc 0.11.0 -3** | 7 049 | 45.79 | **15 394** | **1.53x** | | **zxc 0.11.0 -4** | 6 697 | 42.65 | **15 703** | **1.56x** | | **zxc 0.11.0 -5** | 6 267 | 40.27 | **15 563** | **1.55x** | | **zxc 0.11.0 -6** | 5 620 | 36.28 | **15 490** | **1.54x** | | lz4 1.10.0 (Ref) | 4 783 | 47.60 | 10 048 | 1.00x | | lz4 1.10.0 --fast -17 | 5 623 | 62.15 | 9 047 | 0.90x | | lz4hc 1.10.0 -9 | 4 528 | 36.75 | 12 321 | 1.23x | | lzav 5.7 -1 | 3 877 | 39.94 | 9 707 | 0.97x | | snappy 1.2.2 | 3 264 | 47.85 | 6 822 | 0.68x | | zstd 1.5.7 --fast --1 | 2 538 | 41.01 | 6 189 | 0.62x | | zstd 1.5.7 -1 | 1 806 | 34.53 | 5 230 | 0.52x | | zlib 1.3.1 -1 | 410 | 36.45 | 1 125 | 0.11x | *Higher is better. Captures how much *original* data is delivered per unit of compressed input bandwidth. Formula: `Effective (MB/s) = Decompression Speed × 100 / Compression Ratio (%)`.* *Reading: on Apple M2, ZXC's full level range delivers between **1.53x** and **2.03x** LZ4 effective bandwidth. ZXC -6 (15 490 MB/s, 1.54x LZ4) clearly leads `lz4hc -9` (12 321 MB/s, 1.23x) on this platform — **1.26x more effective bandwidth at equivalent ratio**. Apple Silicon's deep pipelines amplify ZXC's lead at every level.* ### 7.2 Cloud Server Summary (ARM64 / Google Axion Neoverse-V2) | Compressor | Decompression Speed (Ratio vs LZ4) | Compressed Size (Index LZ4=100) (Lower is Better) | | :--- | :--- | :--- | | **zxc 0.11.0 -1** | **2.13x** | **129.22** | | **zxc 0.11.0 -2** | **1.77x** | **112.64** | | **zxc 0.11.0 -3** | **1.24x** | **96.20** | | **zxc 0.11.0 -4** | **1.18x** | **89.60** | | **zxc 0.11.0 -5** | **1.10x** | **84.59** | | **zxc 0.11.0 -6** | **0.99x** | **76.21** | | lz4 1.10.0 --fast -17 | 1.16x | 130.58 | | lz4 1.10.0 (Ref) | 1.00x | 100.00 | | lz4hc 1.10.0 -9 | 0.90x | 77.20 | | lzav 5.7 -1 | 0.65x | 83.91 | | snappy 1.2.2 | 0.54x | 100.53 | | zstd 1.5.7 --fast --1 | 0.54x | 86.16 | | zstd 1.5.7 -1 | 0.39x | 72.55 | | zlib 1.3.1 -1 | 0.09x | 76.58 | **Decompression Efficiency (Cycles per Byte @ 2.6 GHz)** | Compressor. | Cycles/Byte | Performance vs memcpy (*) | | ----------------------- | ----------- | --------------------- | | memcpy | 0.108 | 1.00x (baseline) | | **zxc 0.11.0 -1** | **0.287** | **2.7x** | | **zxc 0.11.0 -2** | **0.346** | **3.2x** | | **zxc 0.11.0 -3** | **0.491** | **4.6x** | | **zxc 0.11.0 -4** | **0.517** | **4.8x** | | **zxc 0.11.0 -5** | **0.555** | **5.2x** | | **zxc 0.11.0 -6** | **0.618** | **5.7x** | | lz4 1.10.0 | 0.610 | 5.7x | | lz4 1.10.0 --fast -17 | 0.525 | 4.9x | | lz4hc 1.10.0 -9 | 0.676 | 6.3x | | lzav 5.7 -1 | 0.943 | 8.8x | | zstd 1.5.7 -1 | 1.581 | 14.7x | | zstd 1.5.7 --fast --1 | 1.133 | 10.5x | | snappy 1.2.2 | 1.124 | 10.4x | | zlib 1.3.1 -1 | 6.667 | 61.9x | *Lower is better. Calculated using Neoverse-V2 base frequency (2.6 GHz). Formula: `Cycles/Byte = 2600 / Decompression Speed (MB/s)`.* **Effective Throughput (Ratio-normalized decode)** This metric expresses how much *original* data is delivered per unit of compressed input bandwidth. Formula: `Effective (MB/s) = Decompression Speed × 100 / Ratio (%)`. It captures the combined benefit of fast decode and good ratio: a smaller compressed file feeds the decoder with less bandwidth pressure on the source (storage / network / inter-core), so each MB of compressed data yields more MB of original data per second of decode work. *Higher is better.* | Compressor | Decode (MB/s) | Ratio (%) | Effective (MB/s) | vs LZ4 | | :--- | ---: | ---: | ---: | ---: | | **zxc 0.11.0 -1** | 9 067 | 61.50 | **14 744** | **1.65x** | | **zxc 0.11.0 -2** | 7 524 | 53.61 | **14 035** | **1.57x** | | **zxc 0.11.0 -3** | 5 297 | 45.79 | **11 569** | **1.29x** | | **zxc 0.11.0 -4** | 5 025 | 42.65 | **11 782** | **1.32x** | | **zxc 0.11.0 -5** | 4 685 | 40.27 | **11 634** | **1.30x** | | **zxc 0.11.0 -6** | 4 205 | 36.28 | **11 591** | **1.30x** | | lz4 1.10.0 (Ref) | 4 259 | 47.60 | 8 948 | 1.00x | | lz4 1.10.0 --fast -17 | 4 951 | 62.15 | 7 966 | 0.89x | | lz4hc 1.10.0 -9 | 3 849 | 36.75 | 10 473 | 1.17x | | lzav 5.7 -1 | 2 757 | 39.94 | 6 903 | 0.77x | | snappy 1.2.2 | 2 313 | 47.85 | 4 834 | 0.54x | | zstd 1.5.7 --fast --1 | 2 295 | 41.01 | 5 596 | 0.63x | | zstd 1.5.7 -1 | 1 645 | 34.53 | 4 764 | 0.53x | | zlib 1.3.1 -1 | 390 | 36.45 | 1 070 | 0.12x | *Higher is better. Captures how much *original* data is delivered per unit of compressed input bandwidth. Formula: `Effective (MB/s) = Decompression Speed × 100 / Compression Ratio (%)`.* *Reading: at ZXC -6, every MB/s of compressed input yields **11 591 MB/s** of original output — **1.11x** more effective bandwidth than `lz4hc -9` at equivalent ratio (36.28 vs 36.75), and **1.30x** more than LZ4 default. ZXC's full level range stays above 1.29x LZ4 across all levels.* ### 7.3 Build Server Summary (x86_64 / AMD EPYC 9B45, Zen 5) | Compressor | Decompression Speed (Ratio vs LZ4) | Compressed Size (Index LZ4=100) (Lower is Better) | | :--- | :--- | :--- | | **zxc 0.11.0 -1** | **2.16x** | **129.22** | | **zxc 0.11.0 -2** | **1.91x** | **112.64** | | **zxc 0.11.0 -3** | **1.19x** | **96.20** | | **zxc 0.11.0 -4** | **1.11x** | **89.60** | | **zxc 0.11.0 -5** | **1.05x** | **84.59** | | **zxc 0.11.0 -6** | **0.94x** | **76.21** | | lz4 1.10.0 --fast -17 | 1.06x | 130.58 | | lz4 1.10.0 (Ref) | 1.00x | 100.00 | | lz4hc 1.10.0 -9 | 0.97x | 77.20 | | lzav 5.7 -1 | 0.72x | 83.91 | | snappy 1.2.2 | 0.42x | 100.63 | | zstd 1.5.7 --fast --1 | 0.48x | 86.16 | | zstd 1.5.7 -1 | 0.37x | 72.55 | | zlib 1.3.1 -1 | 0.08x | 76.58 | **Decompression Efficiency (Cycles per Byte @ 2.1 GHz)** | Compressor. | Cycles/Byte | Performance vs memcpy (*) | | ----------------------- | ----------- | --------------------- | | memcpy | 0.090 | 1.00x (baseline) | | **zxc 0.11.0 -1** | **0.194** | **2.1x** | | **zxc 0.11.0 -2** | **0.219** | **2.4x** | | **zxc 0.11.0 -3** | **0.353** | **3.9x** | | **zxc 0.11.0 -4** | **0.376** | **4.2x** | | **zxc 0.11.0 -5** | **0.399** | **4.4x** | | **zxc 0.11.0 -6** | **0.447** | **5.0x** | | lz4 1.10.0 | 0.419 | 4.6x | | lz4 1.10.0 --fast -17 | 0.396 | 4.4x | | lz4hc 1.10.0 -9 | 0.434 | 4.8x | | lzav 5.7 -1 | 0.579 | 6.4x | | zstd 1.5.7 -1 | 1.124 | 12.5x | | zstd 1.5.7 --fast --1 | 0.872 | 9.7x | | snappy 1.2.2 | 0.992 | 11.0x | | zlib 1.3.1 -1 | 5.426 | 60.2x | *Lower is better. Calculated using AMD EPYC 9B45 base frequency (2.1 GHz). Formula: `Cycles/Byte = 2100 / Decompression Speed (MB/s)`.* **Effective Throughput (Ratio-normalized decode)** | Compressor | Decode (MB/s) | Ratio (%) | Effective (MB/s) | vs LZ4 | | :--- | ---: | ---: | ---: | ---: | | **zxc 0.11.0 -1** | 10 844 | 61.50 | **17 633** | **1.67x** | | **zxc 0.11.0 -2** | 9 597 | 53.61 | **17 902** | **1.70x** | | **zxc 0.11.0 -3** | 5 955 | 45.79 | **13 005** | **1.23x** | | **zxc 0.11.0 -4** | 5 589 | 42.65 | **13 104** | **1.24x** | | **zxc 0.11.0 -5** | 5 259 | 40.27 | **13 059** | **1.24x** | | **zxc 0.11.0 -6** | 4 695 | 36.28 | **12 941** | **1.23x** | | lz4 1.10.0 (Ref) | 5 013 | 47.60 | 10 532 | 1.00x | | lz4 1.10.0 --fast -17 | 5 301 | 62.15 | 8 530 | 0.81x | | lz4hc 1.10.0 -9 | 4 841 | 36.75 | 13 173 | 1.25x | | lzav 5.7 -1 | 3 628 | 39.94 | 9 083 | 0.86x | | snappy 1.2.2 | 2 118 | 47.89 | 4 423 | 0.42x | | zstd 1.5.7 --fast --1 | 2 407 | 41.01 | 5 870 | 0.56x | | zstd 1.5.7 -1 | 1 868 | 34.53 | 5 410 | 0.51x | | zlib 1.3.1 -1 | 387 | 36.45 | 1 062 | 0.10x | *Higher is better. Captures how much *original* data is delivered per unit of compressed input bandwidth. Formula: `Effective (MB/s) = Decompression Speed × 100 / Compression Ratio (%)`.* *Reading: on EPYC 9B45, ZXC's full level range delivers between 1.23x and 1.70x LZ4 effective bandwidth. Note that on this x86_64 platform `lz4hc -9` (1.25x) edges out ZXC -6 (1.23x) on this metric — lz4hc's decode (4 841 MB/s) runs ~3% faster than ZXC -6 (4 695 MB/s) here, while ZXC -6 keeps the ratio advantage (36.28 vs 36.75). The two are practically tied on this platform.* ### 7.4 Production Server Summary (x86_64 / AMD EPYC 7763, Zen 3) | Compressor | Decompression Speed (Ratio vs LZ4) | Compressed Size (Index LZ4=100) (Lower is Better) | | :--- | :--- | :--- | | **zxc 0.11.0 -1** | **2.00x** | **129.22** | | **zxc 0.11.0 -2** | **1.67x** | **112.64** | | **zxc 0.11.0 -3** | **1.11x** | **96.20** | | **zxc 0.11.0 -4** | **1.06x** | **89.60** | | **zxc 0.11.0 -5** | **1.02x** | **84.59** | | **zxc 0.11.0 -6** | **0.90x** | **76.21** | | lz4 1.10.0 --fast -17 | 1.15x | 130.58 | | lz4 1.10.0 (Ref) | 1.00x | 100.00 | | lz4hc 1.10.0 -9 | 0.96x | 77.20 | | lzav 5.7 -1 | 0.74x | 83.91 | | snappy 1.2.2 | 0.45x | 100.63 | | zstd 1.5.7 --fast --1 | 0.46x | 86.16 | | zstd 1.5.7 -1 | 0.34x | 72.55 | | zlib 1.3.1 -1 | 0.09x | 76.58 | **Decompression Efficiency (Cycles per Byte @ 2.45 GHz)** | Compressor. | Cycles/Byte | Performance vs memcpy (*) | | ----------------------- | ----------- | --------------------- | | memcpy | 0.106 | 1.00x (baseline) | | **zxc 0.11.0 -1** | **0.346** | **3.3x** | | **zxc 0.11.0 -2** | **0.415** | **3.9x** | | **zxc 0.11.0 -3** | **0.625** | **5.9x** | | **zxc 0.11.0 -4** | **0.649** | **6.1x** | | **zxc 0.11.0 -5** | **0.676** | **6.4x** | | **zxc 0.11.0 -6** | **0.767** | **7.2x** | | lz4 1.10.0 | 0.691 | 6.5x | | lz4 1.10.0 --fast -17 | 0.599 | 5.6x | | lz4hc 1.10.0 -9 | 0.720 | 6.8x | | lzav 5.7 -1 | 0.939 | 8.9x | | zstd 1.5.7 -1 | 2.007 | 18.9x | | zstd 1.5.7 --fast --1 | 1.507 | 14.2x | | snappy 1.2.2 | 1.540 | 14.5x | | zlib 1.3.1 -1 | 7.470 | 70.4x | *Lower is better. Calculated using AMD EPYC 7763 base frequency (2.45 GHz). Formula: `Cycles/Byte = 2450 / Decompression Speed (MB/s)`.* **Effective Throughput (Ratio-normalized decode)** | Compressor | Decode (MB/s) | Ratio (%) | Effective (MB/s) | vs LZ4 | | :--- | ---: | ---: | ---: | ---: | | **zxc 0.11.0 -1** | 7 077 | 61.50 | **11 507** | **1.54x** | | **zxc 0.11.0 -2** | 5 907 | 53.61 | **11 018** | **1.48x** | | **zxc 0.11.0 -3** | 3 922 | 45.79 | **8 565** | **1.15x** | | **zxc 0.11.0 -4** | 3 775 | 42.65 | **8 851** | **1.19x** | | **zxc 0.11.0 -5** | 3 624 | 40.27 | **8 999** | **1.21x** | | **zxc 0.11.0 -6** | 3 196 | 36.28 | **8 809** | **1.18x** | | lz4 1.10.0 (Ref) | 3 546 | 47.60 | 7 450 | 1.00x | | lz4 1.10.0 --fast -17 | 4 092 | 62.15 | 6 584 | 0.88x | | lz4hc 1.10.0 -9 | 3 401 | 36.75 | 9 254 | 1.24x | | lzav 5.7 -1 | 2 609 | 39.94 | 6 532 | 0.88x | | snappy 1.2.2 | 1 591 | 47.89 | 3 322 | 0.45x | | zstd 1.5.7 --fast --1 | 1 626 | 41.01 | 3 965 | 0.53x | | zstd 1.5.7 -1 | 1 221 | 34.53 | 3 536 | 0.47x | | zlib 1.3.1 -1 | 328 | 36.45 | 900 | 0.12x | *Higher is better. Captures how much *original* data is delivered per unit of compressed input bandwidth. Formula: `Effective (MB/s) = Decompression Speed × 100 / Compression Ratio (%)`.* *Reading: on EPYC 7763 (Zen 3), ZXC's full level range delivers between 1.15x and 1.54x LZ4 effective bandwidth. Like on EPYC 9B45, `lz4hc -9` (1.24x) slightly edges out ZXC -6 (1.18x) at the densest level on this older Zen 3 microarchitecture — its decoder is ~6% faster while the ratio gap stays minimal. ZXC's lead is preserved on the speed-oriented levels (-1 to -2) where the bitstream layout amortizes well over the EPYC 7763 pipeline.* ### 7.5 Benchmarks Results **Figure B**: Decompression Efficiency : Cycles Per Byte Comparaison ![Benchmark Cycles Per Byte](./images/bench-cycles.svg) **Figure C**: Effective Throughput — Ratio-Normalized Decode (vs LZ4 baseline = 1.00x) ![Effective Throughput vs LZ4](./images/bench-effective.svg) #### 7.5.1 ARM64 Architecture (Apple Silicon M2) Benchmarks were conducted using lzbench 2.2.1 (from @inikep), compiled with Clang 21.0.0 using *MOREFLAGS="-march=native"* on macOS Tahoe 26.4 (Build 25E246). The reference hardware is an Apple M2 processor (ARM64). **All performance metrics reflect single-threaded execution on the standard Silesia Corpus and the benchmark made use of [silesia.tar](https://github.com/DataCompression/corpus-collection/tree/main/Silesia-Corpus), which contains tarred files from the Silesia compression corpus.** | Compressor name | Compression| Decompress.| Compr. size | Ratio | Filename | | --------------- | -----------| -----------| ----------- | ----- | -------- | | memcpy | 52866 MB/s | 52887 MB/s | 211947520 |100.00 | 1 files| | **zxc 0.11.0 -1** | 876 MB/s | **12530 MB/s** | 130356444 | **61.50** | 1 files| | **zxc 0.11.0 -2** | 586 MB/s | **10360 MB/s** | 113634139 | **53.61** | 1 files| | **zxc 0.11.0 -3** | 253 MB/s | **7049 MB/s** | 97051816 | **45.79** | 1 files| | **zxc 0.11.0 -4** | 174 MB/s | **6697 MB/s** | 90393215 | **42.65** | 1 files| | **zxc 0.11.0 -5** | 102 MB/s | **6267 MB/s** | 85341643 | **40.27** | 1 files| | **zxc 0.11.0 -6** | 11.8 MB/s | **5620 MB/s** | 76888252 | **36.28** | 1 files| | lz4 1.10.0 | 813 MB/s | 4783 MB/s | 100880800 | 47.60 | 1 files| | lz4 1.10.0 --fast -17 | 1350 MB/s | 5623 MB/s | 131732802 | 62.15 | 1 files| | lz4hc 1.10.0 -9 | 48.2 MB/s | 4528 MB/s | 77884448 | 36.75 | 1 files| | lzav 5.7 -1 | 665 MB/s | 3877 MB/s | 84644732 | 39.94 | 1 files| | snappy 1.2.2 | 880 MB/s | 3264 MB/s | 101415443 | 47.85 | 1 files| | zstd 1.5.7 --fast --1 | 724 MB/s | 2538 MB/s | 86916294 | 41.01 | 1 files| | zstd 1.5.7 -1 | 645 MB/s | 1806 MB/s | 73193704 | 34.53 | 1 files| | zlib 1.3.1 -1 | 150 MB/s | 410 MB/s | 77259029 | 36.45 | 1 files| #### 7.5.2 ARM64 Architecture (Google Axion Neoverse-V2) Benchmarks were conducted using lzbench 2.2.1 (from @inikep), compiled with GCC 14.3.0 using *MOREFLAGS="-march=native"* on Linux 64-bits Debian GNU/Linux 12 (bookworm). The reference hardware is a Google Neoverse-V2 processor (ARM64). **All performance metrics reflect single-threaded execution on the standard Silesia Corpus and the benchmark made use of [silesia.tar](https://github.com/DataCompression/corpus-collection/tree/main/Silesia-Corpus), which contains tarred files from the Silesia compression corpus.** | Compressor name | Compression| Decompress.| Compr. size | Ratio | Filename | | --------------- | -----------| -----------| ----------- | ----- | -------- | | memcpy | 24179 MB/s | 24134 MB/s | 211947520 |100.00 | 1 files| | **zxc 0.11.0 -1** | 868 MB/s | **9067 MB/s** | 130356444 | **61.50** | 1 files| | **zxc 0.11.0 -2** | 586 MB/s | **7524 MB/s** | 113634139 | **53.61** | 1 files| | **zxc 0.11.0 -3** | 238 MB/s | **5297 MB/s** | 97051816 | **45.79** | 1 files| | **zxc 0.11.0 -4** | 165 MB/s | **5025 MB/s** | 90393215 | **42.65** | 1 files| | **zxc 0.11.0 -5** | 96.9 MB/s | **4685 MB/s** | 85341643 | **40.27** | 1 files| | **zxc 0.11.0 -6** | 11.0 MB/s | **4205 MB/s** | 76888252 | **36.28** | 1 files| | lz4 1.10.0 | 732 MB/s | 4259 MB/s | 100880800 | 47.60 | 1 files| | lz4 1.10.0 --fast -17 | 1280 MB/s | 4951 MB/s | 131732802 | 62.15 | 1 files| | lz4hc 1.10.0 -9 | 43.4 MB/s | 3849 MB/s | 77884448 | 36.75 | 1 files| | lzav 5.7 -1 | 562 MB/s | 2757 MB/s | 84644732 | 39.94 | 1 files| | snappy 1.2.2 | 757 MB/s | 2313 MB/s | 101415443 | 47.85 | 1 files| | zstd 1.5.7 --fast --1 | 607 MB/s | 2295 MB/s | 86916294 | 41.01 | 1 files| | zstd 1.5.7 -1 | 525 MB/s | 1645 MB/s | 73193704 | 34.53 | 1 files| | zlib 1.3.1 -1 | 115 MB/s | 390 MB/s | 77259029 | 36.45 | 1 files| #### 7.5.3 x86_64 Architecture (AMD EPYC 9B45) Benchmarks were conducted using lzbench 2.2.1 (from @inikep), compiled with GCC 14.3.0 using *MOREFLAGS="-march=native"* on Linux 64-bits Ubuntu 24.04. The reference hardware is an AMD EPYC 9B45 processor (x86_64). **All performance metrics reflect single-threaded execution on the standard Silesia Corpus and the benchmark made use of [silesia.tar](https://github.com/DataCompression/corpus-collection/tree/main/Silesia-Corpus), which contains tarred files from the Silesia compression corpus.** | Compressor name | Compression| Decompress.| Compr. size | Ratio | Filename | | --------------- | -----------| -----------| ----------- | ----- | -------- | | memcpy | 23351 MB/s | 23292 MB/s | 211947520 |100.00 | 1 files| | **zxc 0.11.0 -1** | 859 MB/s | **10844 MB/s** | 130356444 | **61.50** | 1 files| | **zxc 0.11.0 -2** | 584 MB/s | **9597 MB/s** | 113634139 | **53.61** | 1 files| | **zxc 0.11.0 -3** | 238 MB/s | **5955 MB/s** | 97051816 | **45.79** | 1 files| | **zxc 0.11.0 -4** | 163 MB/s | **5589 MB/s** | 90393215 | **42.65** | 1 files| | **zxc 0.11.0 -5** | 97.0 MB/s | **5259 MB/s** | 85341643 | **40.27** | 1 files| | **zxc 0.11.0 -6** | 11.7 MB/s | **4695 MB/s** | 76888252 | **36.28** | 1 files| | lz4 1.10.0 | 767 MB/s | 5013 MB/s | 100880800 | 47.60 | 1 files| | lz4 1.10.0 --fast -17 | 1280 MB/s | 5301 MB/s | 131732802 | 62.15 | 1 files| | lz4hc 1.10.0 -9 | 45.0 MB/s | 4841 MB/s | 77884448 | 36.75 | 1 files| | lzav 5.7 -1 | 600 MB/s | 3628 MB/s | 84644732 | 39.94 | 1 files| | snappy 1.2.2 | 768 MB/s | 2118 MB/s | 101512076 | 47.89 | 1 files| | zstd 1.5.7 --fast --1 | 656 MB/s | 2407 MB/s | 86916294 | 41.01 | 1 files| | zstd 1.5.7 -1 | 597 MB/s | 1868 MB/s | 73193704 | 34.53 | 1 files| | zlib 1.3.1 -1 | 133 MB/s | 387 MB/s | 77259029 | 36.45 | 1 files| #### 7.5.4 x86_64 Architecture (AMD EPYC 7763, Zen 3) Benchmarks were conducted using lzbench 2.2.1 (from @inikep), compiled with GCC 14.2.0 using *MOREFLAGS="-march=native"* on Linux 64-bits Ubuntu 24.04. The reference hardware is an AMD EPYC 7763 64-Core processor (x86_64, Zen 3, 2.45 GHz). **All performance metrics reflect single-threaded execution on the standard Silesia Corpus and the benchmark made use of [silesia.tar](https://github.com/DataCompression/corpus-collection/tree/main/Silesia-Corpus), which contains tarred files from the Silesia compression corpus.** | Compressor name | Compression| Decompress.| Compr. size | Ratio | Filename | | --------------- | -----------| -----------| ----------- | ----- | -------- | | memcpy | 23023 MB/s | 23087 MB/s | 211947520 |100.00 | 1 files| | **zxc 0.11.0 -1** | 640 MB/s | **7077 MB/s** | 130356444 | **61.50** | 1 files| | **zxc 0.11.0 -2** | 431 MB/s | **5907 MB/s** | 113634139 | **53.61** | 1 files| | **zxc 0.11.0 -3** | 185 MB/s | **3922 MB/s** | 97051816 | **45.79** | 1 files| | **zxc 0.11.0 -4** | 128 MB/s | **3775 MB/s** | 90393215 | **42.65** | 1 files| | **zxc 0.11.0 -5** | 76.5 MB/s | **3624 MB/s** | 85341643 | **40.27** | 1 files| | **zxc 0.11.0 -6** | 8.85 MB/s | **3196 MB/s** | 76888252 | **36.28** | 1 files| | lz4 1.10.0 | 580 MB/s | 3546 MB/s | 100880800 | 47.60 | 1 files| | lz4 1.10.0 --fast -17 | 1015 MB/s | 4092 MB/s | 131732802 | 62.15 | 1 files| | lz4hc 1.10.0 -9 | 33.8 MB/s | 3401 MB/s | 77884448 | 36.75 | 1 files| | lzav 5.7 -1 | 407 MB/s | 2609 MB/s | 84644732 | 39.94 | 1 files| | snappy 1.2.2 | 612 MB/s | 1591 MB/s | 101512076 | 47.89 | 1 files| | zstd 1.5.7 --fast --1 | 443 MB/s | 1626 MB/s | 86916294 | 41.01 | 1 files| | zstd 1.5.7 -1 | 400 MB/s | 1221 MB/s | 73193704 | 34.53 | 1 files| | zlib 1.3.1 -1 | 98.1 MB/s | 328 MB/s | 77259029 | 36.45 | 1 files| ### 7.6 Memory Usage per Compression Context | Block Size | Levels -1 to -5 | Level -6 (DENSITY) | |:---------------------:|----------------:|-------------------:| | 256 KB | ~1.03 MB | ~3.06 MB | | **512 KB** *(default)*| **~1.78 MB** | **~5.84 MB** | | 2 MB *(max)* | ~6.28 MB | ~22.53 MB | *Levels -1 to -5 share the same context layout (LZ77 hash + chain + sequence / literal buffers) and scale linearly with block size. Level -6 (DENSITY) lazily allocates the optimal-parser scratch (per-position DP cost, parent length / offset, packed match-end bitmap), adding ~×3 overhead. Exact values for any (block, level) combination are reproducible via the public API call `zxc_estimate_cctx_size(block_size, level)`.* > **Guideline:** Default 512 KB block keeps cctx under 6 MB even at the densest level (-6) — well within reach for typical server / desktop pipelines. For streaming, embedded, or memory-constrained environments, use `-B 256K` (or smaller) and stick to levels -1 to -5. Level -6 is best reserved for offline encoding pipelines where ratio matters and per-thread RAM is plentiful. ## 8. Strategic Implementation ZXC is designed to adapt to various deployment scenarios by selecting the appropriate compression level: * **Interactive Media & Gaming (Levels 1-2-3)**: Optimized for hard real-time constraints. Ideal for texture streaming and asset loading, offering **~40% faster** load times to minimize latency and frame drops. * **Embedded Systems & Firmware (Levels 3-4-5)**: The sweet spot for maximizing storage density on limited flash memory (e.g., Kernel, Initramfs) while ensuring rapid "instant-on" (XIP-like) boot performance. * **Data Archival (Levels 5-6)**: A high-efficiency alternative for cold storage, providing better compression ratios than LZ4 and significantly faster retrieval speeds than Zstd. **Level 6** (DENSITY) matches LZ4-HC's ratio while keeping ZXC's decode advantage: ideal for write-once / read-many archives where compression time is amortized over many reads. ## 9. Conclusion ZXC redefines asset distribution by prioritizing the end-user experience. Through its asymmetric design and modular architecture, it shifts computational cost to the build pipeline, unlocking unparalleled decompression speeds on ARM devices. This efficiency translates directly into faster load times, reduced battery consumption, and a smoother user experience, making ZXC a best choice for modern, high-performance deployment constraints. zxc-0.11.0/docs/images/000077500000000000000000000000001520102567100145715ustar00rootroot00000000000000zxc-0.11.0/docs/images/bench-arm64.svg000066400000000000000000002526641520102567100173370ustar00rootroot00000000000000zxc-0.11.0/docs/images/bench-bars.svg000066400000000000000000004670401520102567100173310ustar00rootroot00000000000000zxc-0.11.0/docs/images/bench-cycles.svg000066400000000000000000002576721520102567100176740ustar00rootroot00000000000000zxc-0.11.0/docs/images/bench-effective.svg000066400000000000000000003345031520102567100203370ustar00rootroot00000000000000zxc-0.11.0/docs/images/bench-pareto-ratio.svg000066400000000000000000004350701520102567100210060ustar00rootroot00000000000000zxc-0.11.0/docs/man/000077500000000000000000000000001520102567100140775ustar00rootroot00000000000000zxc-0.11.0/docs/man/zxc.1.md000066400000000000000000000137471520102567100154000ustar00rootroot00000000000000# zxc(1) ## NAME **zxc** - High-performance asymmetric lossless compression ## SYNOPSIS **zxc** [*OPTIONS*] [*INPUT-FILE*] [*OUTPUT-FILE*] ## DESCRIPTION **zxc** is a command-line interface for the ZXC compression library, a high-performance lossless compression algorithm optimized for maximum decompression throughput. **zxc** is designed for the *"Write Once, Read Many"* paradigm. It trades compression speed to generate a bitstream specifically structured to maximize decompression speed, effectively offloading complexity from the decoder to the encoder. It aims to provide very high decompression speeds across modern architectures while maintaining competitive compression ratios. ZXC is particularly suited for scenarios such as Game Assets, Firmware or App Bundles where data is compressed once on a build server and decompressed millions of times on user devices. By default, **zxc** compresses a single *INPUT-FILE*. If no *OUTPUT-FILE* is provided, **zxc** will automatically append the `.zxc` extension to the input filename. If no *INPUT-FILE* is provided, **zxc** will read from standard input (`stdin`) and write to standard output (`stdout`). ## STANDARD MODES **-z**, **--compress** : Compress FILE. This is the default mode if no mode is specified. **-d**, **--decompress** : Decompress FILE. **-l**, **--list** : List archive information, including compressed size, uncompressed size, compression ratio, and checksum method. **-t**, **--test** : Test the integrity of a compressed FILE. It decodes the file and verifies its checksum (if present) without writing any output. **-b**, **--bench** [*N*] : Benchmark in-memory performance. Loads the input file entirely into RAM and measures raw algorithm throughput (default duration is 5 seconds). ## OPTIONS **-m**, **--multiple** : Process multiple files at once. When specified, all subsequent non-option arguments are treated as input files. For each input file, a corresponding `.zxc` file is created (or decompressed into its original name). Output cannot be written to standard output (`stdout`) when this mode is enabled. **-r**, **--recursive** : Recursively process directories. When specified, any directory listed as an argument will be traversed, and all regular files within it will be processed (compressed or decompressed). This option implicitly enables `--multiple` mode. **-1**..**-6** : Set the compression level from 1 (fastest compression) to 6 (highest density). - **-1, -2 (Fast):** Optimized for real-time assets or when compression speed is a priority. - **-3 (Default):** Balanced middle-ground offering efficient compression and superior ratio to fast codecs. - **-4, -5 (Compact):** Better ratio than LZ4 with faster decoding than Zstd. Suited for embedded systems and firmware. - **-6 (Max):** Highest ratio tier, matching LZ4-HC while keeping ZXC's decode advantage. Best for archival and write-once / read-many workloads where compression time is amortized over many reads. **-T**, **--threads** *N* : Set the number of threads to use for compression and decompression. A value of `0` means auto-detection based on the number of available CPU cores. **-B**, **--block-size** *SIZE* : Set the compression block size. *SIZE* must be a power of two between **4K** and **2M** (e.g. `4K`, `64K`, `256K`, `512K`, `1M`). Smaller blocks reduce memory usage and improve random-access decompression; larger blocks generally yield better compression ratios. The default is **512K**, tuned for bulk/archival workloads where ratio and decompression throughput matter most. This option is only meaningful during compression; the block size is stored in the archive header and automatically used during decompression. **-C**, **--checksum** : Enable block hashing during compression using the rapidhash algorithm. Recommended for data integrity validation. Checksum verification is automatically performed during extraction when enabled. **-N**, **--no-checksum** : Explicitly disable checksum generation. **-S**, **--seekable** : Append a seek table to the archive during compression. This transforms the file into a random-access format (Seekable Archive), allowing the decoder to instantly locate and decompress specific blocks in `O(1)` time without reading the entire file. Ideal for compressed filesystems, game assets, and log analysis. **-k**, **--keep** : Keep the input file after compression or decompression. (Currently, the input file is preserved by default, but this flag ensures compatibility with future changes). **-f**, **--force** : Force overwrite of the *OUTPUT-FILE* if it already exists. **-c**, **--stdout** : Force writing to standard output (`stdout`), even if it is the console. **-v**, **--verbose** : Enable verbose logging mode. Outputs more detailed information during operations. **-q**, **--quiet** : Enable quiet mode, suppressing all non-error output (such as progress bars or real-time statistics). **-j**, **--json** : Output results in JSON format. This is particularly useful for scripting, benchmarking, and the `--list` mode. ## SPECIAL OPTIONS **-V**, **--version** : Display the version of the zxc library and the compiled architecture information, then exit. **-h**, **--help** : Display a help message and exit. ## EXAMPLES **Compress a file:** zxc data.txt **Compress a file with high density (Level 6, archival):** zxc -6 data.bin **Decompress a file:** zxc -d data.txt.zxc **Compress multiple files independently:** zxc -m file1.txt file2.txt file3.txt **Compress all files in a directory recursively:** zxc -r ./my_folder **Decompress all files in a directory recursively:** zxc -d -r ./my_folder **Decompress a file to standard output:** zxc -dc data.txt.zxc > data.txt **List archive information:** zxc -l data.txt.zxc **Compress with a custom block size (64 KB):** zxc -B 64K data.bin data.zxc **Compress with maximum block size (2 MB):** zxc -6 -B 2M data.bin data.zxc **Run a benchmark for 10 seconds:** zxc -b 10 data.txt ## AUTHORS Written by Bertrand Lebonnois & contributors. ## LICENSE BSD 3-Clause License. zxc-0.11.0/include/000077500000000000000000000000001520102567100140175ustar00rootroot00000000000000zxc-0.11.0/include/zxc.h000066400000000000000000000007671520102567100150060ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ #ifndef ZXC_H #define ZXC_H #include "zxc_buffer.h" // IWYU pragma: keep #include "zxc_constants.h" // IWYU pragma: keep #include "zxc_error.h" // IWYU pragma: keep #include "zxc_opts.h" // IWYU pragma: keep #include "zxc_pstream.h" // IWYU pragma: keep #include "zxc_stream.h" // IWYU pragma: keep #endif // ZXC_Hzxc-0.11.0/include/zxc_buffer.h000066400000000000000000000416011520102567100163270ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /** * @file zxc_buffer.h * @brief Buffer-based (single-shot) compression and decompression API. * * This header exposes the simplest way to use ZXC: pass an entire input buffer * and receive the result in a single output buffer. All functions in this * header are single-threaded and blocking. * * @par Typical usage * @code * // Compress * size_t bound = zxc_compress_bound(src_size); * void *dst = malloc(bound); * zxc_compress_opts_t opts = { .level = ZXC_LEVEL_DEFAULT, .checksum = 1 }; * int64_t csize = zxc_compress(src, src_size, dst, bound, &opts); * * // Decompress * uint64_t orig = zxc_get_decompressed_size(dst, csize); * void *out = malloc(orig); * zxc_decompress_opts_t dopts = { .checksum = 1 }; * int64_t dsize = zxc_decompress(dst, csize, out, orig, &dopts); * @endcode * * @see zxc_stream.h for the streaming (multi-threaded) API. * @see zxc_pstream.h for single-threaded push-based streaming. */ #ifndef ZXC_BUFFER_H #define ZXC_BUFFER_H #include #include #include "zxc_export.h" #include "zxc_opts.h" #ifdef __cplusplus extern "C" { #endif /** * @defgroup library_info Library Information * @brief Runtime-queryable library metadata. * * These functions allow callers (including filesystem integrations) * to discover the supported compression level range and library version at * runtime, without relying on compile-time constants alone. * @{ */ /** * @brief Returns the minimum supported compression level. * * Currently returns @ref ZXC_LEVEL_FASTEST (1). * * @return Minimum compression level value. */ ZXC_EXPORT int zxc_min_level(void); /** * @brief Returns the maximum supported compression level. * * Currently returns @ref ZXC_LEVEL_DENSITY (6). * * @return Maximum compression level value. */ ZXC_EXPORT int zxc_max_level(void); /** * @brief Returns the default compression level. * * Currently returns @ref ZXC_LEVEL_DEFAULT (3). * * @return Default compression level value. */ ZXC_EXPORT int zxc_default_level(void); /** * @brief Returns the human-readable library version string. * * The returned pointer is a compile-time constant and must not be freed. * Example: "0.9.1". * * @return Null-terminated version string. */ ZXC_EXPORT const char* zxc_version_string(void); /** @} */ /* end of library_info */ /** * @defgroup buffer_api Buffer API * @brief Single-shot, buffer-based compression and decompression. * @{ */ /** * @brief Calculates the maximum theoretical compressed size for a given input. * * Useful for allocating output buffers before compression. * Accounts for file headers, block headers, and potential expansion * of incompressible data. * * @param[in] input_size Size of the input data in bytes. * * @return Maximum required buffer size in bytes. */ ZXC_EXPORT uint64_t zxc_compress_bound(const size_t input_size); /** * @brief Compresses a data buffer using the ZXC algorithm. * * This version uses standard size_t types and void pointers. * It executes in a single thread (blocking operation). * It writes the ZXC file header followed by compressed blocks. * * @param[in] src Pointer to the source buffer. * @param[in] src_size Size of the source data in bytes. * @param[out] dst Pointer to the destination buffer. * @param[in] dst_capacity Maximum capacity of the destination buffer. * @param[in] opts Compression options (NULL uses all defaults). * Only @c level, @c block_size, and @c checksum are used. * * @return The number of bytes written to dst (>0 on success), * or a negative zxc_error_t code (e.g., ZXC_ERROR_DST_TOO_SMALL) on failure. */ ZXC_EXPORT int64_t zxc_compress(const void* src, const size_t src_size, void* dst, const size_t dst_capacity, const zxc_compress_opts_t* opts); /** * @brief Decompresses a ZXC compressed buffer. * * This version uses standard size_t types and void pointers. * It executes in a single thread (blocking operation). * It expects a valid ZXC file header followed by compressed blocks. * * @param[in] src Pointer to the source buffer containing compressed data. * @param[in] src_size Size of the compressed data in bytes. * @param[out] dst Pointer to the destination buffer. * @param[in] dst_capacity Capacity of the destination buffer. * @param[in] opts Decompression options (NULL uses all defaults). * Only @c checksum is used. * * @return The number of bytes written to dst (>0 on success), * or a negative zxc_error_t code (e.g., ZXC_ERROR_CORRUPT_DATA) on failure. */ ZXC_EXPORT int64_t zxc_decompress(const void* src, const size_t src_size, void* dst, const size_t dst_capacity, const zxc_decompress_opts_t* opts); /** * @brief Returns the decompressed size stored in a ZXC compressed buffer. * * This function reads the file footer to extract the original uncompressed size * without performing any decompression. Useful for allocating output buffers. * * @param[in] src Pointer to the compressed data buffer. * @param[in] src_size Size of the compressed data in bytes. * * @return The original uncompressed size in bytes, or 0 if the buffer is invalid * or too small to contain a valid ZXC archive. */ ZXC_EXPORT uint64_t zxc_get_decompressed_size(const void* src, const size_t src_size); /* ========================================================================= */ /* Block-Level API (no file framing) */ /* ========================================================================= */ /** * @defgroup block_api Block API * @brief Single-block compression/decompression without file framing. * * These functions compress or decompress a single independent block, producing * only the block header (8 bytes) + compressed payload + optional checksum (4 bytes). * No file header, EOF block, or footer is written. * * This API is designed for filesystem integrations where the filesystem manages its own block * indexing and each block is compressed independently. * * @par Typical usage * @code * // Compress a single filesystem block * zxc_cctx* cctx = zxc_create_cctx(NULL); * zxc_compress_opts_t opts = { .level = 3 }; * size_t bound = zxc_compress_block_bound(block_size); * void *dst = malloc(bound); * int64_t csize = zxc_compress_block(cctx, block, block_size, dst, bound, &opts); * * // Decompress * zxc_dctx* dctx = zxc_create_dctx(); * int64_t dsize = zxc_decompress_block(dctx, dst, csize, out, block_size, NULL); * * zxc_free_cctx(cctx); * zxc_free_dctx(dctx); * @endcode * @{ */ /* Forward declarations for context types (defined below). */ typedef struct zxc_cctx_s zxc_cctx; typedef struct zxc_dctx_s zxc_dctx; /** * @brief Returns the maximum compressed size for a single block. * * Unlike zxc_compress_bound(), this does NOT include file header, * EOF block, or footer overhead. Use this to size the destination * buffer for zxc_compress_block(). * * @param[in] input_size Size of the uncompressed block in bytes. * @return Upper bound on compressed block size, or 0 on overflow. */ ZXC_EXPORT uint64_t zxc_compress_block_bound(size_t input_size); /** * @brief Returns the minimum destination capacity required by * zxc_decompress_block() for a block of @p uncompressed_size bytes. * * The decoder uses speculative (wild-copy) writes on its fast path and * therefore needs a tail pad beyond the declared uncompressed size. * Passing exactly @c uncompressed_size as @c dst_capacity forces the slow * tail path and may trigger @ref ZXC_ERROR_OVERFLOW on some inputs. * * Use this helper to size the destination buffer. The returned value is * guaranteed to enable the fastest decode path without aliasing or * overrun checks tripping. * * @param[in] uncompressed_size Original uncompressed block size in bytes. * @return Minimum @c dst_capacity to pass to zxc_decompress_block(), * or 0 if @p uncompressed_size would overflow. */ ZXC_EXPORT uint64_t zxc_decompress_block_bound(const size_t uncompressed_size); /** * @brief Compresses a single block without file framing. * * Output format: @c block_header(8B) + payload + optional @c checksum(4B). * The output can be decompressed with zxc_decompress_block(). * * @param[in,out] cctx Reusable compression context. * @param[in] src Source data. * @param[in] src_size Source data size in bytes. * @param[out] dst Destination buffer. * @param[in] dst_capacity Capacity of the destination buffer * (use zxc_compress_block_bound() to size). * @param[in] opts Compression options, or NULL for defaults. * Only @c level, @c block_size, and * @c checksum_enabled are used. * * @return Compressed block size in bytes (> 0) on success, * or a negative @ref zxc_error_t code on failure. */ ZXC_EXPORT int64_t zxc_compress_block(zxc_cctx* cctx, const void* src, size_t src_size, void* dst, size_t dst_capacity, const zxc_compress_opts_t* opts); /** * @brief Decompresses a single block produced by zxc_compress_block(). * * @param[in,out] dctx Reusable decompression context. * @param[in] src Compressed block data. * @param[in] src_size Compressed data size in bytes. * @param[out] dst Destination buffer for decompressed data. * @param[in] dst_capacity Capacity of the destination buffer (must be * at least the original uncompressed size). * @param[in] opts Decompression options (NULL for defaults). * Only @c checksum_enabled is used. * * @return Decompressed size in bytes (> 0) on success, * or a negative @ref zxc_error_t code on failure. */ ZXC_EXPORT int64_t zxc_decompress_block(zxc_dctx* dctx, const void* src, size_t src_size, void* dst, size_t dst_capacity, const zxc_decompress_opts_t* opts); /** * @brief Decompresses a single block with a strict-sized destination buffer. * * Identical semantics to zxc_decompress_block() but accepts * @p dst_capacity == @c uncompressed_size (no trailing @c ZXC_DECOMPRESS_TAIL_PAD * required). Intended for integrations whose destination buffer cannot be * oversized (for example, in-place page-aligned decoding). * * This path is slightly slower than zxc_decompress_block() on the same input * because it avoids the wild-copy overshoot that the fast decoder relies on. * Output is bit-identical to zxc_decompress_block(). * * NUM and RAW blocks transparently forward to zxc_decompress_block(); only * GLO/GHI use the strict-tail decoder path. * * @param[in,out] dctx Reusable decompression context. * @param[in] src Compressed block data. * @param[in] src_size Compressed data size in bytes. * @param[out] dst Destination buffer for decompressed data. * @param[in] dst_capacity Capacity of the destination buffer (may equal * the original uncompressed size exactly). * @param[in] opts Decompression options (NULL for defaults). * Only @c checksum_enabled is used. * * @return Decompressed size in bytes (> 0) on success, * or a negative @ref zxc_error_t code on failure. */ ZXC_EXPORT int64_t zxc_decompress_block_safe(zxc_dctx* dctx, const void* src, const size_t src_size, void* dst, const size_t dst_capacity, const zxc_decompress_opts_t* opts); /** * @brief Estimates the peak memory used by compression for a given block & level. * * Returns the total bytes reserved by @ref zxc_compress_block for a block of * @p src_size bytes: all per-chunk working buffers (chain table, literals, * sequence/token/offset/extras buffers) plus the fixed hash tables and * cache-line alignment padding. At @p level >= 6 the value also includes the * `opt_scratch` region (~8.125 x @p src_size bytes) used by the price-based * optimal parser. That region is lazy-allocated on the first level-6 call * and reused across blocks for the lifetime of the cctx. Scales roughly * linearly with @p src_size. * * Intended for integrators that need an accurate memory-budget figure. * * @param[in] src_size Uncompressed block size in bytes. * @param[in] level Compression level (1..6). Levels <= 5 share the same * persistent cctx footprint; level 6 adds the optimal- * parser scratch. * @return Estimated peak cctx memory usage in bytes, or 0 if @p src_size is 0. */ ZXC_EXPORT uint64_t zxc_estimate_cctx_size(size_t src_size, int level); /** @} */ /* end of block_api */ /* ========================================================================= */ /* Reusable Context API (opaque, heap-allocated) */ /* ========================================================================= */ /** * @defgroup context_api Reusable Context API * @brief Opaque, reusable compression / decompression contexts. * * This API eliminates per-call allocation overhead by letting callers retain * a context across multiple operations. The internal layout is hidden behind * an opaque pointer. * * @{ */ /* --- Compression context ------------------------------------------------- */ /** * @brief Creates a new reusable compression context. * * When @p opts is non-NULL the context pre-allocates all internal buffers * using the supplied level, block_size, and checksum_enabled settings. * When @p opts is NULL, allocation is deferred to the first call to * zxc_compress_cctx(). * * The returned context must be freed with zxc_free_cctx(). * * @param[in] opts Compression options for eager init, or NULL for lazy init. * @return Pointer to the new context, or @c NULL on allocation failure. */ ZXC_EXPORT zxc_cctx* zxc_create_cctx(const zxc_compress_opts_t* opts); /** * @brief Frees a compression context and all associated resources. * * It is safe to pass @c NULL; the call is a no-op in that case. * * @param[in] cctx Context to free. */ ZXC_EXPORT void zxc_free_cctx(zxc_cctx* cctx); /** * @brief Compresses data using a reusable context. * * Identical to zxc_compress() but reuses the internal buffers from @p cctx, * avoiding per-call malloc/free overhead. The context automatically * re-initializes when block_size or level changes between calls. * * Options are **sticky**: settings passed via @p opts are remembered and * reused on subsequent calls where @p opts is NULL. The initial sticky * values come from the @p opts passed to zxc_create_cctx(). * * @param[in,out] cctx Reusable compression context. * @param[in] src Source data. * @param[in] src_size Source data size in bytes. * @param[out] dst Destination buffer. * @param[in] dst_capacity Capacity of the destination buffer. * @param[in] opts Compression options, or NULL to reuse * settings from create / last call. * * @return Compressed size in bytes (> 0) on success, * or a negative @ref zxc_error_t code on failure. */ ZXC_EXPORT int64_t zxc_compress_cctx(zxc_cctx* cctx, const void* src, size_t src_size, void* dst, size_t dst_capacity, const zxc_compress_opts_t* opts); /* --- Decompression context ----------------------------------------------- */ /** * @brief Creates a new reusable decompression context. * * @return Pointer to the new context, or @c NULL on allocation failure. */ ZXC_EXPORT zxc_dctx* zxc_create_dctx(void); /** * @brief Frees a decompression context and all associated resources. * * It is safe to pass @c NULL. * * @param[in] dctx Context to free. */ ZXC_EXPORT void zxc_free_dctx(zxc_dctx* dctx); /** * @brief Decompresses data using a reusable context. * * Identical to zxc_decompress() but reuses buffers from @p dctx. * * @param[in,out] dctx Reusable decompression context. * @param[in] src Compressed data. * @param[in] src_size Compressed data size in bytes. * @param[out] dst Destination buffer. * @param[in] dst_capacity Capacity of the destination buffer. * @param[in] opts Decompression options (NULL for defaults). * * @return Decompressed size in bytes (> 0) on success, * or a negative @ref zxc_error_t code on failure. */ ZXC_EXPORT int64_t zxc_decompress_dctx(zxc_dctx* dctx, const void* src, size_t src_size, void* dst, size_t dst_capacity, const zxc_decompress_opts_t* opts); /** @} */ /* end of context_api */ /** @} */ /* end of buffer_api */ #ifdef __cplusplus } #endif #endif // ZXC_BUFFER_Hzxc-0.11.0/include/zxc_constants.h000066400000000000000000000056221520102567100170750ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /** * @file zxc_constants.h * @brief Public constants: library version and compression levels. * * Include this header to query the library version at compile time or to * reference the predefined compression-level constants used throughout the API. */ #ifndef ZXC_CONSTANTS_H #define ZXC_CONSTANTS_H /** * @defgroup version Library Version * @brief Compile-time version information. * @{ */ /** @brief Major version number. */ #define ZXC_VERSION_MAJOR 0 /** @brief Minor version number. */ #define ZXC_VERSION_MINOR 11 /** @brief Patch version number. */ #define ZXC_VERSION_PATCH 0 /** @cond INTERNAL */ #define ZXC_STR_HELPER(x) #x #define ZXC_STR(x) ZXC_STR_HELPER(x) /** @endcond */ /** * @brief Human-readable version string (e.g. "0.7.2"). */ #define ZXC_LIB_VERSION_STR \ ZXC_STR(ZXC_VERSION_MAJOR) \ "." ZXC_STR(ZXC_VERSION_MINOR) "." ZXC_STR(ZXC_VERSION_PATCH) /** @} */ /* end of version */ /** * @defgroup block_size Block Size * @brief Block size constraints for compression. * * Block size must be a power of two in range * [@ref ZXC_BLOCK_SIZE_MIN, @ref ZXC_BLOCK_SIZE_MAX]. * Pass 0 to any API to use @ref ZXC_BLOCK_SIZE_DEFAULT. * @{ */ /** @brief log2(ZXC_BLOCK_SIZE_MIN) - exponent code for minimum block size. */ #define ZXC_BLOCK_SIZE_MIN_LOG2 12 /** @brief log2(ZXC_BLOCK_SIZE_MAX) - exponent code for maximum block size. */ #define ZXC_BLOCK_SIZE_MAX_LOG2 21 /** @brief Default block size (512 KB). */ #define ZXC_BLOCK_SIZE_DEFAULT (512 * 1024) /** @brief Minimum allowed block size (4 KB = 2^12). */ #define ZXC_BLOCK_SIZE_MIN (1U << ZXC_BLOCK_SIZE_MIN_LOG2) /** @brief Maximum allowed block size (2 MB = 2^21). */ #define ZXC_BLOCK_SIZE_MAX (1U << ZXC_BLOCK_SIZE_MAX_LOG2) /** @} */ /* end of block_size */ /** * @defgroup levels Compression Levels * @brief Predefined compression levels for the ZXC library. * * Higher levels trade encoding speed for better compression ratio. * All levels produce data that can be decompressed at the same speed. * @{ */ /** * @brief Enumeration of ZXC compression levels. * * Use one of these constants as the @p level parameter of * zxc_compress() or zxc_stream_compress(). */ typedef enum { ZXC_LEVEL_FASTEST = 1, /**< Fastest compression, best for real-time applications. */ ZXC_LEVEL_FAST = 2, /**< Fast compression, good for real-time applications. */ ZXC_LEVEL_DEFAULT = 3, /**< Recommended: ratio > LZ4, decode speed > LZ4. */ ZXC_LEVEL_BALANCED = 4, /**< Good ratio, good decode speed. */ ZXC_LEVEL_COMPACT = 5, /**< High density. Best for storage/firmware/assets. */ ZXC_LEVEL_DENSITY = 6 /**< Maximum density: Huffman-coded literals on top of COMPACT. */ } zxc_compression_level_t; /** @} */ /* end of levels */ #endif // ZXC_CONSTANTS_H zxc-0.11.0/include/zxc_error.h000066400000000000000000000052061520102567100162100ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /** * @file zxc_error.h * @brief Error codes and error-name lookup for the ZXC library. * * Every public function that can fail returns a value from @ref zxc_error_t. * A return value < 0 indicates an error; use zxc_error_name() to convert * any code to a human-readable string. */ #ifndef ZXC_ERROR_H #define ZXC_ERROR_H #include "zxc_export.h" #ifdef __cplusplus extern "C" { #endif /** * @defgroup error Error Handling * @brief Error codes returned by ZXC library functions. * @{ */ /** * @brief Error codes returned by ZXC library functions. * * All error codes are negative integers. Functions that return int or int64_t * will return these codes on failure. Check with `result < 0` for errors. * * Use zxc_error_name() to get a human-readable string for any error code. */ typedef enum { ZXC_OK = 0, /**< Success (no error). */ /* Memory errors */ ZXC_ERROR_MEMORY = -1, /**< Memory allocation failure. */ /* Buffer/capacity errors */ ZXC_ERROR_DST_TOO_SMALL = -2, /**< Destination buffer too small. */ ZXC_ERROR_SRC_TOO_SMALL = -3, /**< Source buffer too small or truncated input. */ /* Format/header errors */ ZXC_ERROR_BAD_MAGIC = -4, /**< Invalid magic word in file header. */ ZXC_ERROR_BAD_VERSION = -5, /**< Unsupported file format version. */ ZXC_ERROR_BAD_HEADER = -6, /**< Corrupted or invalid header (CRC mismatch). */ ZXC_ERROR_BAD_CHECKSUM = -7, /**< Block or global checksum verification failed. */ /* Data integrity errors */ ZXC_ERROR_CORRUPT_DATA = -8, /**< Corrupted compressed data. */ ZXC_ERROR_BAD_OFFSET = -9, /**< Invalid match offset during decompression. */ ZXC_ERROR_OVERFLOW = -10, /**< Buffer overflow detected during processing. */ /* I/O errors */ ZXC_ERROR_IO = -11, /**< Read/write/seek failure on file. */ ZXC_ERROR_NULL_INPUT = -12, /**< Required input pointer is NULL. */ /* Block errors */ ZXC_ERROR_BAD_BLOCK_TYPE = -13, /**< Unknown or unexpected block type. */ ZXC_ERROR_BAD_BLOCK_SIZE = -14, /**< Invalid block size. */ } zxc_error_t; /** * @brief Returns a human-readable name for the given error code. * * @param[in] code An error code from zxc_error_t (or any integer). * @return A constant string such as "ZXC_OK" or "ZXC_ERROR_MEMORY". * Returns "ZXC_UNKNOWN_ERROR" for unrecognized codes. */ ZXC_EXPORT const char* zxc_error_name(const int code); /** @} */ /* end of error */ #ifdef __cplusplus } #endif #endif /* ZXC_ERROR_H */ zxc-0.11.0/include/zxc_export.h000066400000000000000000000057111520102567100164010ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /** * @file zxc_export.h * @brief Platform-specific symbol visibility macros. * * This header defines the `ZXC_EXPORT`, `ZXC_NO_EXPORT`, and `ZXC_DEPRECATED` * macros that control which symbols are exported from the shared library. * * - Define @c ZXC_STATIC_DEFINE when building or consuming ZXC as a **static** * library to disable import/export annotations. * - When building the shared library the CMake target defines * @c zxc_lib_EXPORTS automatically, selecting `dllexport` / `visibility("default")`. * - When consuming the shared library neither macro is defined, so the header * selects `dllimport` / `visibility("default")`. */ #ifndef ZXC_EXPORT_H #define ZXC_EXPORT_H /** * @defgroup export Symbol Visibility * @brief Macros controlling DLL export/import and deprecation attributes. * @{ */ #ifdef ZXC_STATIC_DEFINE /** * @def ZXC_EXPORT * @brief Marks a symbol as part of the public shared-library API. * * Expands to nothing when building a static library (@c ZXC_STATIC_DEFINE), * to `__declspec(dllexport)` or `__declspec(dllimport)` on Windows, or * to `__attribute__((visibility("default")))` on GCC/Clang. */ #define ZXC_EXPORT /** * @def ZXC_NO_EXPORT * @brief Marks a symbol as hidden (not exported from the shared library). * * Expands to nothing for static builds or Windows, and to * `__attribute__((visibility("hidden")))` on GCC/Clang. */ #define ZXC_NO_EXPORT #else /* shared library */ #ifndef ZXC_EXPORT #ifdef zxc_lib_EXPORTS /* Building the library */ #ifdef _WIN32 #define ZXC_EXPORT __declspec(dllexport) #else #define ZXC_EXPORT __attribute__((visibility("default"))) #endif #else /* Consuming the library */ #ifdef _WIN32 #define ZXC_EXPORT __declspec(dllimport) #else #define ZXC_EXPORT __attribute__((visibility("default"))) #endif #endif #endif #ifndef ZXC_NO_EXPORT #ifdef _WIN32 #define ZXC_NO_EXPORT #else #define ZXC_NO_EXPORT __attribute__((visibility("hidden"))) #endif #endif #endif /* ZXC_STATIC_DEFINE */ #ifndef ZXC_DEPRECATED /** * @def ZXC_DEPRECATED * @brief Marks a symbol as deprecated. * * The compiler will emit a warning when a deprecated symbol is referenced. * Expands to `__declspec(deprecated)` on MSVC or * `__attribute__((__deprecated__))` on GCC/Clang. */ #ifdef _WIN32 #define ZXC_DEPRECATED __declspec(deprecated) #else #define ZXC_DEPRECATED __attribute__((__deprecated__)) #endif #endif /** * @def ZXC_DEPRECATED_EXPORT * @brief Combines `ZXC_EXPORT` with `ZXC_DEPRECATED`. */ #ifndef ZXC_DEPRECATED_EXPORT #define ZXC_DEPRECATED_EXPORT ZXC_EXPORT ZXC_DEPRECATED #endif /** * @def ZXC_DEPRECATED_NO_EXPORT * @brief Combines `ZXC_NO_EXPORT` with `ZXC_DEPRECATED`. */ #ifndef ZXC_DEPRECATED_NO_EXPORT #define ZXC_DEPRECATED_NO_EXPORT ZXC_NO_EXPORT ZXC_DEPRECATED #endif /** @} */ /* end of export */ #endif /* ZXC_EXPORT_H */ zxc-0.11.0/include/zxc_opts.h000066400000000000000000000063351520102567100160500ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /** * @file zxc_opts.h * @brief Shared option structures for the ZXC compression APIs. * * Defines @ref zxc_compress_opts_t, @ref zxc_decompress_opts_t and * @ref zxc_progress_callback_t. These types are consumed by every public * ZXC API (one-shot buffer, multi-threaded @c FILE* streaming, push * streaming, seekable). * * This header is never used in isolation: include the API header you * actually use (@c zxc_buffer.h, @c zxc_stream.h, @c zxc_pstream.h, ...) * which pulls this one in transitively. */ #ifndef ZXC_OPTS_H #define ZXC_OPTS_H #include #include #ifdef __cplusplus extern "C" { #endif /** * @brief Progress callback function type. * * This callback is invoked periodically during compression/decompression to report * progress. It is called from the writer thread after each block is processed. * * @param[in] bytes_processed Total input bytes processed so far. * @param[in] bytes_total Total input bytes to process (0 if unknown, e.g., stdin). * @param[in] user_data User-provided context pointer (passed through from API call). * * @note The callback should be fast and non-blocking. Avoid heavy I/O or mutex locks. */ typedef void (*zxc_progress_callback_t)(uint64_t bytes_processed, uint64_t bytes_total, const void* user_data); /** * @brief Options for streaming compression. * * Zero-initialise for safe defaults: level 0 maps to ZXC_LEVEL_DEFAULT (3), * block_size 0 maps to ZXC_BLOCK_SIZE_DEFAULT, n_threads 0 means * auto-detect, and all other fields are disabled. * * @code * zxc_compress_opts_t opts = { .level = ZXC_LEVEL_COMPACT }; * zxc_stream_compress(f_in, f_out, &opts); * @endcode */ typedef struct { int n_threads; /**< Worker thread count (0 = auto-detect CPU cores). */ int level; /**< Compression level 1-5 (0 = default, ZXC_LEVEL_DEFAULT). */ size_t block_size; /**< Block size in bytes (0 = default ZXC_BLOCK_SIZE_DEFAULT). Must be power of 2, [4KB - 2MB]. */ int checksum_enabled; /**< 1 to enable per-block and global checksums, 0 to disable. */ int seekable; /**< 1 to append a seek table for random-access decompression. */ zxc_progress_callback_t progress_cb; /**< Optional progress callback (NULL to disable). */ void* user_data; /**< User context pointer passed to progress_cb. */ } zxc_compress_opts_t; /** * @brief Options for streaming decompression. * * Zero-initialise for safe defaults. * * @code * zxc_decompress_opts_t opts = { .checksum = 1 }; * zxc_stream_decompress(f_in, f_out, &opts); * @endcode */ typedef struct { int n_threads; /**< Worker thread count (0 = auto-detect CPU cores). */ int checksum_enabled; /**< 1 to verify per-block and global checksums, 0 to skip. */ zxc_progress_callback_t progress_cb; /**< Optional progress callback (NULL to disable). */ void* user_data; /**< User context pointer passed to progress_cb. */ } zxc_decompress_opts_t; #ifdef __cplusplus } #endif #endif // ZXC_OPTS_H zxc-0.11.0/include/zxc_pstream.h000066400000000000000000000263151520102567100165360ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /** * @file zxc_pstream.h * @brief Push-based, single-threaded streaming compression and decompression. * * This header exposes a non-blocking, caller-driven streaming API, the * counterpart of the @c FILE*-based @ref zxc_stream_compress / @ref * zxc_stream_decompress. Where the @c FILE* API takes ownership of the * pipeline (it reads until EOF and writes until done), the push API hands the * control back to the caller: feed input chunks when available, drain output * chunks when ready, finalise on demand. * * Use this API when you need to integrate ZXC into: * - a callback-driven library; * - an asynchronous event loop; * - a network protocol that streams data without seeking (HTTP chunked * transfer, gRPC, custom binary protocols); * - any pipeline where you cannot block on a @c FILE*. * * The API is single-threaded: one context is processed by one thread at a * time. For multi-threaded compression of a single file end-to-end, use * @ref zxc_stream_compress instead. * * @par Compression usage * @code * zxc_compress_opts_t opts = { .level = 3, .checksum_enabled = 1 }; * zxc_cstream* cs = zxc_cstream_create(&opts); * * uint8_t in_buf[64*1024], out_buf[64*1024]; * zxc_outbuf_t out = { out_buf, sizeof out_buf, 0 }; * * ssize_t n; * while ((n = read_some(in_buf, sizeof in_buf)) > 0) { * zxc_inbuf_t in = { in_buf, (size_t)n, 0 }; * while (in.pos < in.size) { * int64_t r = zxc_cstream_compress(cs, &out, &in); * if (r < 0) goto fatal; * if (out.pos > 0) { write_to_sink(out_buf, out.pos); out.pos = 0; } * } * } * * int64_t pending; * do { * pending = zxc_cstream_end(cs, &out); * if (pending < 0) goto fatal; * if (out.pos > 0) { write_to_sink(out_buf, out.pos); out.pos = 0; } * } while (pending > 0); * * zxc_cstream_free(cs); * @endcode * * @see zxc_stream.h for the multi-threaded @c FILE*-based pipeline. * @see zxc_buffer.h for one-shot in-memory compression. */ #ifndef ZXC_PSTREAM_H #define ZXC_PSTREAM_H #include #include #include "zxc_export.h" #include "zxc_opts.h" #ifdef __cplusplus extern "C" { #endif /** * @defgroup pstream Push Streaming API * @brief Caller-driven, single-threaded streaming compression and decompression. * @{ */ /** * @brief Input buffer descriptor for push streaming. * * The caller fills @c src with bytes to feed in and sets @c size to their * count. The library advances @c pos as it consumes input; the caller must * not modify @c pos between calls. */ typedef struct { const void* src; /**< Caller-owned input bytes. */ size_t size; /**< Total bytes available in @c src. */ size_t pos; /**< Bytes already consumed by the library (in/out). */ } zxc_inbuf_t; /** * @brief Output buffer descriptor for push streaming. * * The caller provides a writable region of capacity @c size starting at * @c dst. The library writes starting at @c dst+pos and advances @c pos by * the number of bytes produced. The caller drains @c [dst, dst+pos) and * resets @c pos to 0 between rounds (or grows @c size). */ typedef struct { void* dst; /**< Caller-owned output region. */ size_t size; /**< Total capacity available at @c dst. */ size_t pos; /**< Bytes already produced by the library (in/out). */ } zxc_outbuf_t; /* Opaque streaming contexts. */ typedef struct zxc_cstream_s zxc_cstream; typedef struct zxc_dstream_s zxc_dstream; /* ===== Compression =================================================== */ /** * @brief Creates a push compression stream. * * All settings from @p opts are copied into the context. After this call, * the @p opts struct may be freed or reused. * * Only @c level, @c block_size, and @c checksum_enabled are honoured. * @c n_threads is ignored (this API is single-threaded; use * @ref zxc_stream_compress for the multi-threaded @c FILE* pipeline). * * If @p opts is not @c NULL, the honoured fields must contain valid values. * Invalid option values (for example an unsupported @c block_size) cause * stream creation to fail. * * @param[in] opts Compression options, or @c NULL for all defaults. * @return Allocated context to be released with @ref zxc_cstream_free, * or @c NULL if stream creation fails due to memory allocation * failure or invalid option values in @p opts. */ ZXC_EXPORT zxc_cstream* zxc_cstream_create(const zxc_compress_opts_t* opts); /** * @brief Releases a compression stream and all internal buffers. * * Safe to call with @c NULL (no-op). * * @param[in] cs Stream returned by @ref zxc_cstream_create. */ ZXC_EXPORT void zxc_cstream_free(zxc_cstream* cs); /** * @brief Pushes input bytes into the stream and drains compressed output. * * Reads from @c in->src starting at @c in->pos, writes to @c out->dst * starting at @c out->pos, advancing both as data flows. Each call makes as * much progress as either buffer allows in a single visit: * * - emits the file header on the first invocation (16 B); * - copies input into the internal block accumulator; * - whenever the accumulator is full, compresses one block and writes it * into @p out (up to @c out->size); * - returns when @p in is fully consumed *and* no more compressed bytes are * pending, or when @p out has no room left. * * The function is fully reentrant: if @p out fills mid-block, the next call * resumes draining from where the previous left off. Safe to call with * @c in->size == in->pos (drain-only mode). * * @par Errors * On failure the context becomes errored (sticky): every subsequent call to * @ref zxc_cstream_compress / @ref zxc_cstream_end returns the same negative * code without doing further work. Only @ref zxc_cstream_free is safe. * * @param[in,out] cs Compression stream. * @param[in,out] out Output descriptor; @c pos is advanced by produced bytes. * @param[in,out] in Input descriptor; @c pos is advanced by consumed bytes. * * @return @c 0 if @p in was fully consumed and no compressed bytes remain * pending in the internal staging area; * @c >0 number of bytes still pending, drain @p out and call again * with the same (or new) input; * @c <0 a @ref zxc_error_t code. */ ZXC_EXPORT int64_t zxc_cstream_compress(zxc_cstream* cs, zxc_outbuf_t* out, zxc_inbuf_t* in); /** * @brief Finalises the stream: flushes pending data, writes EOF block + footer. * * Must be called after the last @ref zxc_cstream_compress invocation to * produce a valid ZXC file. Like @ref zxc_cstream_compress, this function * is reentrant: if @p out fills before everything is drained, it returns a * positive count and the caller drains and calls again. * * After @ref zxc_cstream_end returns @c 0, the stream is in DONE state and * any further call returns @c ZXC_ERROR_NULL_INPUT (use @ref * zxc_cstream_free to release). * * @param[in,out] cs Compression stream. * @param[in,out] out Output descriptor. * * @return @c 0 finalisation complete (file is now valid); * @c >0 bytes still pending, drain @p out and call again; * @c <0 a @ref zxc_error_t code. */ ZXC_EXPORT int64_t zxc_cstream_end(zxc_cstream* cs, zxc_outbuf_t* out); /** * @brief Suggested input chunk size for best throughput. * * Equal to the configured block size (default 512 KB). The caller may * supply any input chunk; this is purely a performance hint. * * @param[in] cs Compression stream. * @return Suggested @c in_buf capacity in bytes, or 0 if @p cs is @c NULL. */ ZXC_EXPORT size_t zxc_cstream_in_size(const zxc_cstream* cs); /** * @brief Suggested output chunk size to never trigger a partial drain. * * Sized to hold one full compressed block plus framing overhead. Smaller * outputs work but may force the caller into an extra drain loop. * * @param[in] cs Compression stream. * @return Suggested @c out_buf capacity in bytes, or 0 if @p cs is @c NULL. */ ZXC_EXPORT size_t zxc_cstream_out_size(const zxc_cstream* cs); /* ===== Decompression ================================================= */ /** * @brief Creates a push decompression stream. * * All settings from @p opts are copied into the context. Only * @c checksum_enabled is honoured (controls whether per-block and global * checksums are verified when present). * * @param[in] opts Decompression options, or @c NULL for defaults. * @return Allocated context to be released with @ref zxc_dstream_free, * or @c NULL on allocation failure. */ ZXC_EXPORT zxc_dstream* zxc_dstream_create(const zxc_decompress_opts_t* opts); /** * @brief Releases a decompression stream. Safe to call with @c NULL. * * @param[in] ds Stream returned by @ref zxc_dstream_create. */ ZXC_EXPORT void zxc_dstream_free(zxc_dstream* ds); /** * @brief Pushes compressed input and drains decompressed output. * * Internally runs a parser state machine: file header -> per-block * (header + payload + optional checksum) -> EOF block -> optional SEK block -> * file footer. Each call makes as much progress as @p in and @p out allow. * * @par End of stream * When the decoder reaches the file footer and validates it, the stream * enters DONE state. Subsequent calls return @c 0 without producing more * output, even if extra bytes remain in @p in (those trailing bytes are * silently ignored, the caller may use the residual @c in->pos to detect * how much real data was consumed). * * @par Errors * Sticky: once a negative code is returned, further calls keep returning it. * * @param[in,out] ds Decompression stream. * @param[in,out] out Output descriptor; @c pos advanced by produced bytes. * @param[in,out] in Input descriptor; @c pos advanced by consumed bytes. * * @return @c >0 number of decompressed bytes written into @p out this call; * @c 0 stream complete (DONE) or no progress possible (caller should * feed more input); * @c <0 a @ref zxc_error_t code. */ ZXC_EXPORT int64_t zxc_dstream_decompress(zxc_dstream* ds, zxc_outbuf_t* out, zxc_inbuf_t* in); /** * @brief Reports whether the decoder has fully consumed a valid stream. * * Returns @c 1 iff the parser has reached the file footer **and** validated * it. Callers that have finished feeding input use this to detect truncated * streams: if @ref zxc_dstream_decompress returns @c 0 with no output and * @ref zxc_dstream_finished returns @c 0, the input ended prematurely. * * @param[in] ds Decompression stream. * @return @c 1 if DONE, @c 0 otherwise (including errored). */ ZXC_EXPORT int zxc_dstream_finished(const zxc_dstream* ds); /** * @brief Suggested input chunk size for the decompressor. * * @param[in] ds Decompression stream. * @return Suggested @c in_buf capacity in bytes, or 0 if @p ds is @c NULL. */ ZXC_EXPORT size_t zxc_dstream_in_size(const zxc_dstream* ds); /** * @brief Suggested output chunk size for the decompressor. * * Sized to hold at least one full decompressed block. * * @param[in] ds Decompression stream. * @return Suggested @c out_buf capacity in bytes, or 0 if @p ds is @c NULL. */ ZXC_EXPORT size_t zxc_dstream_out_size(const zxc_dstream* ds); /** @} */ /* end of pstream */ #ifdef __cplusplus } #endif #endif /* ZXC_PSTREAM_H */ zxc-0.11.0/include/zxc_sans_io.h000066400000000000000000000264371520102567100165230ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /** * @file zxc_sans_io.h * @brief Low-level, sans-I/O compression primitives. * * This header exposes the building blocks used by the higher-level buffer and * streaming APIs. It is intended for callers who need to drive compression or * decompression themselves - for example, to integrate ZXC into a custom I/O * pipeline or an event loop. * * A typical usage pattern is: * -# Allocate and initialise a context with zxc_cctx_init(). * -# Write the file header with zxc_write_file_header(). * -# Compress or decompress individual blocks via the chunk wrappers. * -# Write the footer with zxc_write_file_footer(). * -# Release the context with zxc_cctx_free(). */ #ifndef ZXC_SANS_IO_H #define ZXC_SANS_IO_H #include #include #include "zxc_export.h" #ifdef __cplusplus extern "C" { #endif /** * @defgroup sans_io Sans-IO API * @brief Low-level primitives for building custom compression drivers. * @{ */ /** * @struct zxc_cctx_t * @brief Compression Context structure. * * This structure holds the state and buffers required for the compression * process. It is designed to be reused across multiple blocks or calls to avoid * the overhead of repeated memory allocations. * * **Key Fields:** * - `hash_table`: Stores epoch-tagged positions (`ZXC_LZ_HASH_SIZE` * 4 bytes). * - `hash_tags`: Stores 8-bit tags for fast rejection (`ZXC_LZ_HASH_SIZE` * 1 byte). * - `chain_table`: Handles collisions by storing the *previous* occurrence of a * hash. This forms a linked list for each hash bucket, allowing us to * traverse history. * - `epoch`: Used for "Lazy Hash Table Invalidation". Instead of * `ZXC_MEMSET`ing the entire hash table (which is slow) for every block, we * store `(epoch << 16) | offset`. If the stored epoch doesn't match the current * `ctx->epoch`, the entry is considered invalid/empty. * * hash_table Pointer to the hash table used for LZ77 match finding. * chain_table Pointer to the chain table for collision resolution. * memory_block Pointer to the single allocation block containing all buffers. * epoch Current epoch counter for lazy hash table invalidation. * buf_extras Pointer to the buffer for extra lengths (LL >= 15 or ML >= 15). * buf_offsets Pointer to the buffer for offsets. * buf_tokens Pointer to the buffer for token sequences. * literals Pointer to the buffer for raw literal bytes. * lit_buffer Pointer to a scratch buffer for literal processing (e.g., * RLE decoding). * lit_buffer_cap Current capacity of the literal scratch buffer. * checksum_enabled Flag indicating if checksums should be computed. * compression_level The configured compression level. */ typedef struct { /* Hot zone: random access / high frequency. * Kept at the start to ensure they reside in the first cache line (64 bytes). */ uint32_t* hash_table; /**< Hash table for LZ77 match positions (epoch|pos). */ uint8_t* hash_tags; /**< Split tag table for fast match rejection (8-bit tags). */ uint16_t* chain_table; /**< Chain table for collision resolution. */ void* memory_block; /**< Single allocation block owner. */ uint32_t epoch; /**< Current epoch for lazy hash table invalidation. */ /* Warm zone: sequential access per sequence. */ uint32_t* buf_sequences; /**< Buffer for sequence records (packed: LL(8)|ML(8)|Offset(16)). */ uint8_t* buf_tokens; /**< Buffer for token sequences. */ uint16_t* buf_offsets; /**< Buffer for offsets. */ uint8_t* buf_extras; /**< Buffer for extra lengths (vbytes for LL/ML). */ uint8_t* literals; /**< Buffer for literal bytes. */ /* Cold zone: configuration / scratch / resizeable. */ uint8_t* lit_buffer; /**< Scratch buffer for literals (RLE). */ size_t lit_buffer_cap; /**< Current capacity of the scratch buffer. */ uint8_t* work_buf; /**< Padded scratch buffer for buffer-API decompression. */ size_t work_buf_cap; /**< Capacity of the work buffer. */ uint8_t* opt_scratch; /**< Optimal-parser DP scratch (level >= 6 only, lazy-allocated, packs dp/parent_len/parent_off/actions). Also reused as transient scratch for the length-limited Huffman code-length builder. */ size_t opt_scratch_cap; /**< Current capacity of opt_scratch in bytes. */ int checksum_enabled; /**< 1 if checksum calculation/verification is enabled. */ int compression_level; /**< Compression level. */ /* Block-size derived parameters (computed once at init). */ size_t chunk_size; /**< Effective block size in bytes. */ uint32_t offset_bits; /**< log2(chunk_size) - governs epoch_mark shift. */ uint32_t offset_mask; /**< (1U << offset_bits) - 1 */ uint32_t max_epoch; /**< 1U << (32 - offset_bits) */ } zxc_cctx_t; /** * @brief Initializes a ZXC compression context. * * Sets up the internal state required for compression operations, allocating * necessary buffers based on the chunk size and compression level. * * @param[out] ctx Pointer to the ZXC compression context structure to initialize. * @param[in] chunk_size The size of the data chunk to be compressed. This * determines the allocation size for various internal buffers. * @param[in] mode The operation mode (1 for compression, 0 for decompression). * @param[in] level The desired compression level to be stored in the context. * @param[in] checksum_enabled 1 to enable checksums, 0 to disable. * @return ZXC_OK on success, or a negative zxc_error_t code (e.g., ZXC_ERROR_MEMORY) if memory * allocation fails. */ ZXC_EXPORT int zxc_cctx_init(zxc_cctx_t* ctx, const size_t chunk_size, const int mode, const int level, const int checksum_enabled); /** * @brief Frees resources associated with a ZXC compression context. * * This function releases all internal buffers and tables associated with the * given ZXC compression context structure. It does not free the context pointer * itself, only its members. * * @param[in,out] ctx Pointer to the compression context to clean up. */ ZXC_EXPORT void zxc_cctx_free(zxc_cctx_t* ctx); /** * @brief Writes the standard ZXC file header to a destination buffer. * * This function stores the magic word (little-endian) and the version number * into the provided buffer. It ensures the buffer has sufficient capacity * before writing. * * @param[out] dst The destination buffer where the header will be written. * @param[in] dst_capacity The total capacity of the destination buffer in bytes. * @param[in] chunk_size The block size to encode in the header. * @param[in] has_checksum Flag indicating whether the checksum bit should be set. * @return The number of bytes written (ZXC_FILE_HEADER_SIZE) on success, * or ZXC_ERROR_DST_TOO_SMALL if the destination capacity is insufficient. */ ZXC_EXPORT int zxc_write_file_header(uint8_t* dst, const size_t dst_capacity, const size_t chunk_size, const int has_checksum); /** * @brief Validates and reads the ZXC file header from a source buffer. * * This function checks if the provided source buffer is large enough to contain * a ZXC file header and verifies that the magic word and version number match * the expected ZXC format specifications. * * @param[in] src Pointer to the source buffer containing the file data. * @param[in] src_size Size of the source buffer in bytes. * @param[out] out_block_size Optional pointer to receive the recommended block size. * @param[out] out_has_checksum Optional pointer to receive the checksum flag. * @return ZXC_OK on success, or a negative error code (e.g., ZXC_ERROR_SRC_TOO_SMALL, * ZXC_ERROR_BAD_MAGIC, ZXC_ERROR_BAD_VERSION). */ ZXC_EXPORT int zxc_read_file_header(const uint8_t* src, const size_t src_size, size_t* out_block_size, int* out_has_checksum); /** * @struct zxc_block_header_t * @brief Represents the on-disk header structure for a ZXC block (8 bytes). * * This structure contains metadata required to parse and decompress a block. * Note: raw_size is not stored in the header; decoders derive it from Section * Descriptors within the compressed payload. */ typedef struct { uint8_t block_type; /**< Block type (see @ref zxc_block_type_t). */ uint8_t block_flags; /**< Flags (e.g., checksum presence). */ uint8_t reserved; /**< Reserved for future protocol extensions. */ uint8_t header_crc; /**< Header integrity checksum (1 byte). */ uint32_t comp_size; /**< Compressed size excluding this header. */ } zxc_block_header_t; /** * @brief Encodes a block header into the destination buffer. * * This function serializes the contents of a `zxc_block_header_t` structure * into a byte array in little-endian format. It ensures the destination buffer * has sufficient capacity before writing. * * @param[out] dst Pointer to the destination buffer where the header will be * written. * @param[in] dst_capacity The total size of the destination buffer in bytes. * @param[in] bh Pointer to the source block header structure containing the data to * write. * * @return The number of bytes written (ZXC_BLOCK_HEADER_SIZE) on success, * or ZXC_ERROR_DST_TOO_SMALL if the destination buffer capacity is insufficient. */ ZXC_EXPORT int zxc_write_block_header(uint8_t* dst, const size_t dst_capacity, const zxc_block_header_t* bh); /** * @brief Reads and parses a ZXC block header from a source buffer. * * This function extracts the block type, flags, reserved fields, compressed * size, and raw size from the first `ZXC_BLOCK_HEADER_SIZE` bytes of the source * buffer. It handles endianness conversion for multi-byte fields (Little * Endian). * * @param[in] src Pointer to the source buffer containing the block data. * @param[in] src_size The size of the source buffer in bytes. * @param[out] bh Pointer to a `zxc_block_header_t` structure where the parsed * header information will be stored. * * @return ZXC_OK on success, or ZXC_ERROR_SRC_TOO_SMALL if the source buffer is smaller * than the required block header size. */ ZXC_EXPORT int zxc_read_block_header(const uint8_t* src, const size_t src_size, zxc_block_header_t* bh); /** * @brief Writes the ZXC file footer. * * The footer stores the original uncompressed size and an optional global * checksum. It is always @c ZXC_FILE_FOOTER_SIZE (12) bytes. * * @param[out] dst Destination buffer. * @param[in] dst_capacity Capacity of destination buffer. * @param[in] src_size Original uncompressed size of the data. * @param[in] global_hash Global checksum hash (if enabled). * @param[in] checksum_enabled Flag indicating if checksum is enabled. * @return Number of bytes written (12) on success, or ZXC_ERROR_DST_TOO_SMALL on failure. */ ZXC_EXPORT int zxc_write_file_footer(uint8_t* dst, const size_t dst_capacity, const uint64_t src_size, const uint32_t global_hash, const int checksum_enabled); /** @} */ /* end of sans_io */ #ifdef __cplusplus } #endif #endif // ZXC_SANS_IO_Hzxc-0.11.0/include/zxc_seekable.h000066400000000000000000000176561520102567100166460ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /** * @file zxc_seekable.h * @brief Seekable compression and random-access decompression API. * * This header provides functions to produce seekable ZXC archives and to * decompress arbitrary byte ranges without reading the entire file. * * A seekable archive embeds a Seek Table block (block_type = @c ZXC_BLOCK_SEK) * after the EOF block, recording the compressed size of every block. * The table is detected at read time by deriving @c num_blocks from the file * footer's total decompressed size and the header's block size, then seeking * backward to validate the SEK block header. * Standard (non-seekable) decompressors ignore the seek table entirely. * * @par Creating a seekable archive * @code * zxc_compress_opts_t opts = { .level = 3, .seekable = 1 }; * int64_t csize = zxc_compress(src, src_size, dst, dst_cap, &opts); * @endcode * * @par Random-access decompression * @code * zxc_seekable* s = zxc_seekable_open(compressed, csize); * int64_t n = zxc_seekable_decompress_range(s, out, out_cap, offset, len); * zxc_seekable_free(s); * @endcode */ #ifndef ZXC_SEEKABLE_H #define ZXC_SEEKABLE_H #include #include #include #include "zxc_export.h" #ifdef __cplusplus extern "C" { #endif /** * @defgroup seekable_api Seekable API * @brief Random-access compression and decompression. * @{ */ /* ========================================================================= */ /* Seekable Reader (Random-Access Decompression) */ /* ========================================================================= */ /** * @brief Opaque handle for a seekable ZXC archive. * * Created by zxc_seekable_open() or zxc_seekable_open_file(). * Must be freed with zxc_seekable_free(). */ typedef struct zxc_seekable_s zxc_seekable; /** * @brief Opens a seekable archive from a memory buffer. * * Parses the seek table from the end of the buffer and builds the internal * block index. The buffer must remain valid for the lifetime of the handle. * * @param[in] src Pointer to the compressed data. * @param[in] src_size Size of the compressed data in bytes. * @return Handle on success, or @c NULL if the buffer does not contain a * valid seekable archive (e.g. missing seek block, bad block type). */ ZXC_EXPORT zxc_seekable* zxc_seekable_open(const void* src, const size_t src_size); /** * @brief Opens a seekable archive from a FILE*. * * The file must be seekable (not stdin/pipe). The current file position * is saved and restored after parsing the seek table. The FILE* must * remain open for the lifetime of the handle. * * @param[in] f File opened in "rb" mode. * @return Handle on success, or @c NULL on error. */ ZXC_EXPORT zxc_seekable* zxc_seekable_open_file(FILE* f); /** * @brief Returns the total number of blocks in the seekable archive. * * @param[in] s Seekable handle. * @return Number of data blocks (excluding EOF). */ ZXC_EXPORT uint32_t zxc_seekable_get_num_blocks(const zxc_seekable* s); /** * @brief Returns the total decompressed size of the seekable archive. * * @param[in] s Seekable handle. * @return Total decompressed size in bytes. */ ZXC_EXPORT uint64_t zxc_seekable_get_decompressed_size(const zxc_seekable* s); /** * @brief Returns the compressed size of a specific block. * * This is the "on-disk" size including block header, payload, and optional * per-block checksum. * * @param[in] s Seekable handle. * @param[in] block_idx Zero-based block index. * @return Compressed block size, or 0 if @p block_idx is out of range. */ ZXC_EXPORT uint32_t zxc_seekable_get_block_comp_size(const zxc_seekable* s, const uint32_t block_idx); /** * @brief Returns the decompressed size of a specific block. * * @param[in] s Seekable handle. * @param[in] block_idx Zero-based block index. * @return Decompressed block size, or 0 if @p block_idx is out of range. */ ZXC_EXPORT uint32_t zxc_seekable_get_block_decomp_size(const zxc_seekable* s, const uint32_t block_idx); /** * @brief Decompresses an arbitrary byte range from the original data. * * Only the blocks overlapping [@p offset, @p offset + @p len) are read and * decompressed. This is the core random-access primitive. * * @param[in,out] s Seekable handle. * @param[out] dst Destination buffer. * @param[in] dst_capacity Capacity of @p dst (must be >= @p len). * @param[in] offset Byte offset into the original uncompressed data. * @param[in] len Number of bytes to decompress. * @return Number of bytes written to @p dst (== @p len on success), * or a negative @ref zxc_error_t code on failure. */ ZXC_EXPORT int64_t zxc_seekable_decompress_range(zxc_seekable* s, void* dst, const size_t dst_capacity, const uint64_t offset, const size_t len); /** * @brief Multi-threaded variant of zxc_seekable_decompress_range(). * * Decompresses blocks in parallel using a fork-join thread pool. Each worker * thread owns its own decompression context and reads compressed data via * @c pread() (POSIX) or @c ReadFile() (Windows) for lock-free concurrent I/O. * * Falls back to single-threaded mode when @p n_threads <= 1 or when the * requested range spans a single block. * * @param[in,out] s Seekable handle. * @param[out] dst Destination buffer. * @param[in] dst_capacity Capacity of @p dst (must be >= @p len). * @param[in] offset Byte offset into the original uncompressed data. * @param[in] len Number of bytes to decompress. * @param[in] n_threads Number of worker threads (0 = auto-detect CPU cores). * @return Number of bytes written to @p dst (== @p len on success), * or a negative @ref zxc_error_t code on failure. */ ZXC_EXPORT int64_t zxc_seekable_decompress_range_mt(zxc_seekable* s, void* dst, const size_t dst_capacity, const uint64_t offset, const size_t len, int n_threads); /** * @brief Frees a seekable handle and all associated resources. * * Safe to call with @c NULL. * * @param[in] s Handle to free. */ ZXC_EXPORT void zxc_seekable_free(zxc_seekable* s); /* ========================================================================= */ /* Seek Table Writer (low-level) */ /* ========================================================================= */ /** * @brief Writes a seek table to the destination buffer. * * This is a low-level helper used internally by the seekable compression * paths. It writes: block_header(8) + N entries(4 each). * Each entry stores only @c comp_size; decompressed sizes are derived at * read time from the file header's block_size. * * @param[out] dst Destination buffer. * @param[in] dst_capacity Capacity of @p dst in bytes. * @param[in] comp_sizes Array of compressed block sizes. * @param[in] num_blocks Number of blocks. * @return Number of bytes written, or a negative @ref zxc_error_t on failure. */ ZXC_EXPORT int64_t zxc_write_seek_table(uint8_t* dst, const size_t dst_capacity, const uint32_t* comp_sizes, const uint32_t num_blocks); /** * @brief Returns the encoded size of a seek table for the given block count. * * @param[in] num_blocks Number of blocks. * @return Total byte size of the seek table. */ ZXC_EXPORT size_t zxc_seek_table_size(const uint32_t num_blocks); /** @} */ /* end of seekable_api */ #ifdef __cplusplus } #endif #endif /* ZXC_SEEKABLE_H */ zxc-0.11.0/include/zxc_stream.h000066400000000000000000000062101520102567100163460ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /** * @file zxc_stream.h * @brief Multi-threaded streaming compression and decompression API. * * This header provides the streaming driver that reads from a @c FILE* input and * writes compressed (or decompressed) output to a @c FILE*. Internally the * driver uses an asynchronous Producer-Consumer pipeline via a ring buffer to * separate I/O from CPU-intensive work: * * 1. **Reader thread** - reads chunks from `f_in`. * 2. **Worker threads** - compress/decompress chunks in parallel. * 3. **Writer thread** - orders the results and writes them to `f_out`. * * @see zxc_buffer.h for the simple one-shot buffer API. * @see zxc_pstream.h for single-threaded push-based streaming. */ #ifndef ZXC_STREAM_H #define ZXC_STREAM_H #include #include #include "zxc_export.h" #include "zxc_opts.h" #ifdef __cplusplus extern "C" { #endif /** * @defgroup stream_api Streaming API * @brief Multi-threaded, FILE*-based compression and decompression. * @{ */ /** * @brief Compresses data from an input stream to an output stream. * * This function sets up a multi-threaded pipeline: * 1. Reader Thread: Reads chunks from f_in. * 2. Worker Threads: Compress chunks in parallel (LZ77 + Bitpacking). * 3. Writer Thread: Orders the processed chunks and writes them to f_out. * * @param[in] f_in Input file stream (must be opened in "rb" mode). * @param[out] f_out Output file stream (must be opened in "wb" mode). * @param[in] opts Compression options (NULL uses all defaults). * * @return Total compressed bytes written, or a negative zxc_error_t code (e.g., * ZXC_ERROR_IO) if an error occurred. */ ZXC_EXPORT int64_t zxc_stream_compress(FILE* f_in, FILE* f_out, const zxc_compress_opts_t* opts); /** * @brief Decompresses data from an input stream to an output stream. * * Uses the same pipeline architecture as compression to maximize throughput. * * @param[in] f_in Input file stream (must be opened in "rb" mode). * @param[out] f_out Output file stream (must be opened in "wb" mode). * @param[in] opts Decompression options (NULL uses all defaults). * * @return Total decompressed bytes written, or a negative zxc_error_t code (e.g., * ZXC_ERROR_BAD_HEADER) if an error occurred. */ ZXC_EXPORT int64_t zxc_stream_decompress(FILE* f_in, FILE* f_out, const zxc_decompress_opts_t* opts); /** * @brief Returns the decompressed size stored in a ZXC compressed file. * * This function reads the file footer to extract the original uncompressed size * without performing any decompression. The file position is restored after reading. * * @param[in] f_in Input file stream (must be opened in "rb" mode). * * @return The original uncompressed size in bytes, or a negative zxc_error_t code (e.g., * ZXC_ERROR_BAD_MAGIC) if the file is invalid or an I/O error occurred. */ ZXC_EXPORT int64_t zxc_stream_get_decompressed_size(FILE* f_in); /** @} */ /* end of stream_api */ #ifdef __cplusplus } #endif #endif // ZXC_STREAM_Hzxc-0.11.0/libzxc.pc.in000066400000000000000000000005171520102567100146230ustar00rootroot00000000000000prefix=@CMAKE_INSTALL_PREFIX@ exec_prefix=${prefix} libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ Name: libzxc Description: High-performance asymmetric lossless compression License: BSD 3-Clause Version: @PROJECT_VERSION@ Libs: -L${libdir} -lzxc Libs.private: -pthread Cflags: -I${includedir}zxc-0.11.0/src/000077500000000000000000000000001520102567100131635ustar00rootroot00000000000000zxc-0.11.0/src/cli/000077500000000000000000000000001520102567100137325ustar00rootroot00000000000000zxc-0.11.0/src/cli/main.c000066400000000000000000001455011520102567100150300ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /** * @file main.c * @brief Command Line Interface (CLI) entry point for the ZXC compression tool. * * This file handles argument parsing, file I/O setup, platform-specific * compatibility layers (specifically for Windows), and the execution of * compression, decompression, or benchmarking modes. */ #include #include #include #include #include #include "../../include/zxc_buffer.h" #include "../../include/zxc_constants.h" #include "../../include/zxc_stream.h" #include "../lib/zxc_internal.h" #if defined(_WIN32) #define ZXC_OS "windows" #elif defined(__APPLE__) #define ZXC_OS "darwin" #elif defined(__linux__) #define ZXC_OS "linux" #else #define ZXC_OS "unknown" #endif #if defined(__x86_64__) || defined(_M_AMD64) #define ZXC_ARCH "x86_64" #elif defined(__aarch64__) || defined(_M_ARM64) #define ZXC_ARCH "arm64" #else #define ZXC_ARCH "unknown" #endif #ifdef _WIN32 // Windows Implementation #include #include #include #include // Map POSIX macros to MSVC equivalents #define F_OK 0 #define access _access #define isatty _isatty #define fileno _fileno #define unlink _unlink #define fseeko _fseeki64 #define ftello _ftelli64 /** * @brief Returns the current monotonic time in seconds using Windows * Performance Counter. * @return double Time in seconds. */ static double zxc_now(void) { LARGE_INTEGER frequency; LARGE_INTEGER count; QueryPerformanceFrequency(&frequency); QueryPerformanceCounter(&count); return (double)count.QuadPart / frequency.QuadPart; } struct option { const char* name; int has_arg; int* flag; int val; }; #define no_argument 0 #define required_argument 1 #define optional_argument 2 char* optarg = NULL; int optind = 1; int optopt = 0; /** * @brief Minimal implementation of getopt_long for Windows. * Handles long options (--option) and short options (-o). */ static int getopt_long(int argc, char* const argv[], const char* optstring, const struct option* longopts, int* longindex) { if (optind >= argc) return -1; char* curr = argv[optind]; if (curr[0] == '-' && curr[1] == '-') { char* name_end = strchr(curr + 2, '='); const size_t name_len = name_end ? (size_t)(name_end - (curr + 2)) : strlen(curr + 2); const struct option* p = longopts; while (p && p->name) { const size_t opt_len = strlen(p->name); if (name_len == opt_len && strncmp(curr + 2, p->name, name_len) == 0) { optind++; if (p->has_arg == required_argument) { if (name_end) optarg = name_end + 1; else if (optind < argc) optarg = argv[optind++]; else return '?'; } else if (p->has_arg == optional_argument) { if (name_end) optarg = name_end + 1; else optarg = NULL; } if (p->flag) { *p->flag = p->val; return 0; } return p->val; } p++; } return '?'; } if (curr[0] == '-') { char c = curr[1]; optind++; const char* os = strchr(optstring, c); if (!os) return '?'; if (os[1] == ':') { if (os[2] == ':') { // Optional argument (::) if (curr[2] != '\0') optarg = curr + 2; else optarg = NULL; } else { // Required argument (:) if (curr[2] != '\0') optarg = curr + 2; else if (optind < argc) optarg = argv[optind++]; else return '?'; } } return c; } return -1; } #else // POSIX / Linux / macOS Implementation #include #include #include #include #include #include #include #include #include #include #include #include /** * @brief Returns the current monotonic time in seconds using clock_gettime. * @return double Time in seconds. */ static double zxc_now(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return ts.tv_sec + ts.tv_nsec / 1e9; } #endif /** * @brief Validates and resolves the input file path to prevent directory traversal * and ensure it is a regular file. * * @param[in] path The raw input path from command line. * @param[out] resolved_buffer Buffer to store resolved path (needs sufficient size). * @param[in] buffer_size Size of the resolved_buffer. * @return 0 on success, -1 on error. */ static int zxc_validate_input_path(const char* path, char* resolved_buffer, size_t buffer_size) { #ifdef _WIN32 if (!_fullpath(resolved_buffer, path, buffer_size)) { return -1; } DWORD attr = GetFileAttributesA(resolved_buffer); if (attr == INVALID_FILE_ATTRIBUTES || (attr & FILE_ATTRIBUTE_DIRECTORY)) { // Not a valid file or is a directory errno = (attr == INVALID_FILE_ATTRIBUTES) ? ENOENT : EISDIR; return -1; } return 0; #else char* const res = realpath(path, NULL); if (!res) { // realpath failed (e.g. file does not exist) return -1; } struct stat st; if (stat(res, &st) != 0) { free(res); return -1; } if (!S_ISREG(st.st_mode)) { free(res); errno = EISDIR; // Generic error for non-regular file return -1; } const size_t len = strlen(res); if (len >= buffer_size) { free(res); errno = ENAMETOOLONG; return -1; } memcpy(resolved_buffer, res, len + 1); free(res); return 0; #endif } /** * @brief Validates and resolves the output file path. * * @param[in] path The raw output path. * @param[out] resolved_buffer Buffer to store resolved path. * @param[in] buffer_size Size of the resolved_buffer. * @return 0 on success, -1 on error. */ static int zxc_validate_output_path(const char* path, char* resolved_buffer, size_t buffer_size) { #ifdef _WIN32 if (!_fullpath(resolved_buffer, path, buffer_size)) return -1; DWORD attr = GetFileAttributesA(resolved_buffer); if (attr != INVALID_FILE_ATTRIBUTES && (attr & FILE_ATTRIBUTE_DIRECTORY)) { errno = EISDIR; return -1; } return 0; #else // POSIX output path validation char* const temp_path = strdup(path); if (!temp_path) return -1; char* const temp_path2 = strdup(path); if (!temp_path2) { free(temp_path); return -1; } // Split into dir and base char* const dir = dirname(temp_path); // Note: dirname may modify string or return static const char* const base = basename(temp_path2); char* const resolved_dir = realpath(dir, NULL); if (!resolved_dir) { // Parent directory must exist free(temp_path); free(temp_path2); return -1; } struct stat st; if (stat(resolved_dir, &st) != 0 || !S_ISDIR(st.st_mode)) { free(resolved_dir); free(temp_path); free(temp_path2); errno = EISDIR; return -1; } // Reconstruct valid path: resolved_dir / base // Ensure we don't overflow buffer const int written = snprintf(resolved_buffer, buffer_size, "%s/%s", resolved_dir, base); free(resolved_dir); free(temp_path); free(temp_path2); if (written < 0 || (size_t)written >= buffer_size) { errno = ENAMETOOLONG; return -1; } return 0; #endif } // CLI Logging Helpers static int g_quiet = 0; static int g_verbose = 0; /** * @brief Standard logging function. Respects the global quiet flag. */ static void zxc_log(const char* fmt, ...) { if (g_quiet) return; va_list args; va_start(args, fmt); vfprintf(stderr, fmt, args); va_end(args); } /** * @brief Verbose logging function. Only prints if verbose is enabled and quiet * is disabled. */ static void zxc_log_v(const char* fmt, ...) { if (!g_verbose || g_quiet) return; va_list args; va_start(args, fmt); vfprintf(stderr, fmt, args); va_end(args); } // OS-specific helpers for directory checks #ifdef _WIN32 static int zxc_is_directory(const char* path) { DWORD attr = GetFileAttributesA(path); return (attr != INVALID_FILE_ATTRIBUTES && (attr & FILE_ATTRIBUTE_DIRECTORY)); } #else static int zxc_is_directory(const char* path) { struct stat st; if (stat(path, &st) == 0) { return S_ISDIR(st.st_mode); } return 0; } #endif typedef enum { MODE_COMPRESS, MODE_DECOMPRESS, MODE_BENCHMARK, MODE_INTEGRITY, MODE_LIST } zxc_mode_t; enum { OPT_VERSION = 1000, OPT_HELP }; // Forward declaration for recursive mode static int process_single_file(const char* in_path, const char* out_path_override, zxc_mode_t mode, int num_threads, int keep_input, int force, int to_stdout, int checksum, int level, size_t block_size, int json_output, int seekable); // Forward declaration for processing directory static int process_directory(const char* dir_path, zxc_mode_t mode, int num_threads, int keep_input, int force, int to_stdout, int checksum, int level, size_t block_size, int json_output, int seekable); // OS-specific implementation of directory processing static int process_directory(const char* dir_path, zxc_mode_t mode, int num_threads, int keep_input, int force, int to_stdout, int checksum, int level, size_t block_size, int json_output, int seekable) { int overall_ret = 0; #ifdef _WIN32 char search_path[MAX_PATH]; snprintf(search_path, sizeof(search_path), "%s\\*", dir_path); WIN32_FIND_DATAA find_data; HANDLE hFind = FindFirstFileA(search_path, &find_data); if (hFind == INVALID_HANDLE_VALUE) { zxc_log("Error opening directory '%s'\n", dir_path); return 1; } do { if (strcmp(find_data.cFileName, ".") == 0 || strcmp(find_data.cFileName, "..") == 0) { continue; } char full_path[MAX_PATH]; snprintf(full_path, sizeof(full_path), "%s\\%s", dir_path, find_data.cFileName); if (find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { overall_ret |= process_directory(full_path, mode, num_threads, keep_input, force, to_stdout, checksum, level, block_size, json_output, seekable); } else { // Check if it ends with .zxc to skip if compressing to avoid double compression if (mode == MODE_COMPRESS) { const size_t len = strlen(full_path); if (len >= 4 && strcmp(full_path + len - 4, ".zxc") == 0) { continue; // Skip already compressed files in recursive compression } } overall_ret |= process_single_file(full_path, NULL, mode, num_threads, keep_input, force, to_stdout, checksum, level, block_size, json_output, seekable); } } while (FindNextFileA(hFind, &find_data) != 0); FindClose(hFind); #else DIR* const dir = opendir(dir_path); if (!dir) { zxc_log("Error opening directory '%s': %s\n", dir_path, strerror(errno)); return 1; } const struct dirent* entry; while ((entry = readdir(dir)) != NULL) { if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) { continue; } const size_t path_len = strlen(dir_path) + 1 + strlen(entry->d_name) + 1; char* const full_path = malloc(path_len); if (!full_path) { zxc_log("Error allocating memory for path in directory '%s'\n", dir_path); continue; } const int n = snprintf(full_path, path_len, "%s/%s", dir_path, entry->d_name); if (n < 0 || (size_t)n >= path_len) { zxc_log("Error: path too long in directory '%s'\n", dir_path); free(full_path); continue; } struct stat st; if (stat(full_path, &st) == 0) { if (S_ISDIR(st.st_mode)) { overall_ret |= process_directory(full_path, mode, num_threads, keep_input, force, to_stdout, checksum, level, block_size, json_output, seekable); } else if (S_ISREG(st.st_mode)) { // Check if it ends with .zxc to skip if compressing to avoid double compression if (mode == MODE_COMPRESS) { const size_t len = strlen(full_path); if (len >= 4 && strcmp(full_path + len - 4, ".zxc") == 0) { free(full_path); continue; // Skip already compressed files in recursive compression } } overall_ret |= process_single_file(full_path, NULL, mode, num_threads, keep_input, force, to_stdout, checksum, level, block_size, json_output, seekable); } } free(full_path); } closedir(dir); #endif return overall_ret; } void print_help(const char* app) { printf("Usage: %s [] []...\n\n", app); printf( "Standard Modes:\n" " -z, --compress Compress FILE {default}\n" " -d, --decompress Decompress FILE (or stdin -> stdout)\n" " -l, --list List archive information\n" " -t, --test Test compressed FILE integrity\n" " -b, --bench [N] Benchmark in-memory (N=seconds, default 5)\n\n" "Batch Processing:\n" " -m, --multiple Multiple input files\n" " -r, --recursive Operate recursively on directories\n\n" "Special Options:\n" " -V, --version Show version information\n" " -h, --help Show this help message\n\n" "Options:\n" " -1..-6 Compression level {3}\n" " -B, --block-size Block size: 4K..2M, power of 2 {512K}\n" " -T, --threads N Number of threads (0=auto)\n" " -C, --checksum Enable checksum {default}\n" " -N, --no-checksum Disable checksum\n" " -S, --seekable Append seek table for random-access decompression\n" " -k, --keep Keep input file\n" " -f, --force Force overwrite\n" " -c, --stdout Write to stdout\n" " -v, --verbose Verbose mode\n" " -q, --quiet Quiet mode\n" " -j, --json JSON output (benchmark mode)\n"); } void print_version(void) { char sys_info[256]; #ifdef _WIN32 snprintf(sys_info, sizeof(sys_info), "%s-%s", ZXC_ARCH, ZXC_OS); #else struct utsname buffer; if (uname(&buffer) == 0) snprintf(sys_info, sizeof(sys_info), "%s-%s-%s", ZXC_ARCH, ZXC_OS, buffer.release); else snprintf(sys_info, sizeof(sys_info), "%s-%s", ZXC_ARCH, ZXC_OS); #endif printf("zxc v%s (%s) by Bertrand Lebonnois & al.\nBSD 3-Clause License\n", ZXC_LIB_VERSION_STR, sys_info); } /** * @brief Formats a byte size into human-readable TB/GB/MB/KB/B format (Base 1000). */ static void format_size_decimal(uint64_t bytes, char* buf, size_t buf_size) { const double TB = 1000.0 * 1000.0 * 1000.0 * 1000.0; const double GB = 1000.0 * 1000.0 * 1000.0; const double MB = 1000.0 * 1000.0; const double KB = 1000.0; if ((double)bytes >= TB) snprintf(buf, buf_size, "%.1f TB", (double)bytes / TB); else if ((double)bytes >= GB) snprintf(buf, buf_size, "%.1f GB", (double)bytes / GB); else if ((double)bytes >= MB) snprintf(buf, buf_size, "%.1f MB", (double)bytes / MB); else if ((double)bytes >= KB) snprintf(buf, buf_size, "%.1f KB", (double)bytes / KB); else snprintf(buf, buf_size, "%llu B", (unsigned long long)bytes); } /** * @brief Progress context for CLI progress bar display. */ typedef struct { double start_time; const char* operation; // "Compressing" or "Decompressing" uint64_t total_size; // Pre-determined total size (0 if unknown) } progress_ctx_t; /** * @brief Progress callback for CLI progress bar. * * Displays a real-time progress bar during compression/decompression. * Shows percentage, processed/total size, and throughput speed. * * Format: [==========> ] 45% | 4.5 GB/10.0 GB | 156 MB/s */ static void cli_progress_callback(uint64_t bytes_processed, uint64_t bytes_total, const void* user_data) { const progress_ctx_t* pctx = (const progress_ctx_t*)user_data; if (!pctx) return; // Use pre-determined total size from context (not the parameter) const uint64_t total = pctx->total_size; const double now = zxc_now(); const double elapsed = now - pctx->start_time; // Calculate throughput speed double speed_mbps = 0.0; if (elapsed > 0.1) // Avoid division by zero for very fast operations speed_mbps = (double)bytes_processed / (1000.0 * 1000.0) / elapsed; // Clear line and move cursor to beginning fprintf(stderr, "\r\033[K"); if (total > 0) { // Known size: show percentage bar int percent = (int)((bytes_processed * 100) / total); if (percent > 100) percent = 100; const int bar_width = 20; int filled = (percent * bar_width) / 100; fprintf(stderr, "%s [", pctx->operation); for (int i = 0; i < bar_width; i++) { if (i < filled) fprintf(stderr, "="); else if (i == filled) fprintf(stderr, ">"); else fprintf(stderr, " "); } fprintf(stderr, "] %d%% | ", percent); char proc_str[32], total_str[32]; format_size_decimal(bytes_processed, proc_str, sizeof(proc_str)); format_size_decimal(total, total_str, sizeof(total_str)); fprintf(stderr, "%s/%s | %.1f MB/s", proc_str, total_str, speed_mbps); } else { // Unknown size (stdin): just show bytes processed char proc_str[32]; format_size_decimal(bytes_processed, proc_str, sizeof(proc_str)); fprintf(stderr, "%s [Processing...] %s | %.1f MB/s", pctx->operation, proc_str, speed_mbps); } fflush(stderr); } /** * @brief Lists the contents of a ZXC archive. * * Reads the file header and footer to display: * - Compressed size * - Uncompressed size * - Compression ratio * - Checksum method * - Filename * * In verbose mode, displays additional header information. * * @param[in] path Path to the ZXC archive file. * @param[in] json_output If 1, output JSON format. * @return 0 on success, 1 on error. */ static int zxc_list_archive(const char* path, int json_output) { char resolved_path[4096]; if (zxc_validate_input_path(path, resolved_path, sizeof(resolved_path)) != 0) { fprintf(stderr, "Error: Invalid input file '%s': %s\n", path, strerror(errno)); return 1; } FILE* f = fopen(resolved_path, "rb"); if (!f) { fprintf(stderr, "Error: Cannot open '%s': %s\n", path, strerror(errno)); return 1; } // Get file size if (fseeko(f, 0, SEEK_END) != 0) { fclose(f); fprintf(stderr, "Error: Cannot seek in file\n"); return 1; } const long long file_size = ftello(f); // Use public API to get decompressed size const int64_t uncompressed_size = zxc_stream_get_decompressed_size(f); if (uncompressed_size < 0) { fclose(f); fprintf(stderr, "Error: Not a valid ZXC archive\n"); return 1; } // Read header for format info (rewind after API call) uint8_t header[ZXC_FILE_HEADER_SIZE]; if (fseeko(f, 0, SEEK_SET) != 0 || fread(header, 1, ZXC_FILE_HEADER_SIZE, f) != ZXC_FILE_HEADER_SIZE) { fclose(f); fprintf(stderr, "Error: Cannot read file header\n"); return 1; } // Extract header fields const uint8_t format_version = header[4]; const size_t block_units = header[5] ? header[5] : 64; // Default 64 units = 256KB // Read footer for checksum info uint8_t footer[ZXC_FILE_FOOTER_SIZE]; if (fseeko(f, file_size - ZXC_FILE_FOOTER_SIZE, SEEK_SET) != 0 || fread(footer, 1, ZXC_FILE_FOOTER_SIZE, f) != ZXC_FILE_FOOTER_SIZE) { fclose(f); fprintf(stderr, "Error: Cannot read file footer\n"); return 1; } fclose(f); // Parse checksum (if non-zero, checksum was enabled) const uint32_t stored_checksum = footer[8] | ((uint32_t)footer[9] << 8) | ((uint32_t)footer[10] << 16) | ((uint32_t)footer[11] << 24); const char* checksum_method = (stored_checksum != 0) ? "RapidHash" : "-"; // Calculate ratio (uncompressed / compressed, e.g., 2.5 means 2.5x compression) const double ratio = (file_size > 0) ? ((double)uncompressed_size / (double)file_size) : 0.0; // Format sizes char comp_str[32], uncomp_str[32]; format_size_decimal((uint64_t)file_size, comp_str, sizeof(comp_str)); format_size_decimal((uint64_t)uncompressed_size, uncomp_str, sizeof(uncomp_str)); if (json_output) { // JSON mode printf( "{\n" " \"filename\": \"%s\",\n" " \"compressed_size_bytes\": %lld,\n" " \"uncompressed_size_bytes\": %lld,\n" " \"compression_ratio\": %.3f,\n" " \"format_version\": %u,\n" " \"block_size_kb\": %zu,\n" " \"checksum_method\": \"%s\",\n" " \"checksum_value\": \"0x%08X\"\n" "}\n", path, (long long)file_size, (long long)uncompressed_size, ratio, format_version, block_units * 4, (stored_checksum != 0) ? "RapidHash" : "none", stored_checksum); } else if (g_verbose) { // Verbose mode: detailed vertical layout printf( "\nFile: %s\n" "-----------------------\n" "Block Format: %u\n" "Block Units: %zu (x 4KB)\n" "Checksum Method: %s\n", path, format_version, block_units, (stored_checksum != 0) ? "RapidHash" : "None"); if (stored_checksum != 0) printf("Checksum Value: 0x%08X\n", stored_checksum); printf( "-----------------------\n" "Comp. Size: %s\n" "Uncomp. Size: %s\n" "Ratio: %.2f\n", comp_str, uncomp_str, ratio); } else { // Normal mode: table format printf("\n %12s %12s %5s %-10s %s\n", "Compressed", "Uncompressed", "Ratio", "Checksum", "Filename"); printf(" %12s %12s %5.2f %-10s %s\n", comp_str, uncomp_str, ratio, checksum_method, path); } return 0; } static int process_single_file(const char* in_path, const char* out_path_override, zxc_mode_t mode, int num_threads, int keep_input, int force, int to_stdout, int checksum_enabled, int level, size_t block_size, int json_output, int seekable) { FILE* f_in = stdin; FILE* f_out = stdout; char resolved_in_path[4096] = {0}; char out_path[4096] = {0}; char resolved_out_path[4096] = {0}; int use_stdin = 1, use_stdout = 0; int created_out_file = 0; int overall_ret = 0; if (in_path && strcmp(in_path, "-") != 0) { if (zxc_validate_input_path(in_path, resolved_in_path, sizeof(resolved_in_path)) != 0) { zxc_log("Error: Invalid input file '%s': %s\n", in_path, strerror(errno)); return 1; } f_in = fopen(resolved_in_path, "rb"); if (!f_in) { zxc_log("Error open input %s: %s\n", resolved_in_path, strerror(errno)); return 1; } use_stdin = 0; } else { use_stdin = 1; use_stdout = 1; // Default to stdout if reading from stdin in_path = NULL; } if (mode == MODE_INTEGRITY) { use_stdout = 0; f_out = NULL; } else if (to_stdout) { use_stdout = 1; } else if (!use_stdin) { // Auto-generate output filename if input is a file and no output specified if (out_path_override) { snprintf(out_path, sizeof(out_path), "%s", out_path_override); } else if (mode == MODE_COMPRESS) { snprintf(out_path, sizeof(out_path), "%s.zxc", in_path); } else { const size_t len = strlen(in_path); if (len > 4 && !strcmp(in_path + len - 4, ".zxc")) { const size_t base_len = len - 4; if (base_len >= sizeof(out_path)) { zxc_log("Error: Output path too long\n"); if (f_in) fclose(f_in); return 1; } memcpy(out_path, in_path, base_len); out_path[base_len] = '\0'; } else { zxc_log("Error: Cannot determine output filename: '%s' does not end with .zxc\n", in_path); if (f_in) fclose(f_in); return 1; } } use_stdout = 0; } // Safety check: prevent overwriting input file if (mode != MODE_INTEGRITY && !use_stdin && !use_stdout && strcmp(in_path ? in_path : "", out_path) == 0) { zxc_log("Error: Input and output filenames are identical for '%s'.\n", in_path); if (f_in) fclose(f_in); return 1; } // Open output file if not writing to stdout if (!use_stdout && mode != MODE_INTEGRITY) { if (zxc_validate_output_path(out_path, resolved_out_path, sizeof(resolved_out_path)) != 0) { zxc_log("Error: Invalid output path '%s': %s\n", out_path, strerror(errno)); if (f_in) fclose(f_in); return 1; } if (!force && access(resolved_out_path, F_OK) == 0) { zxc_log("Output exists. Use -f to overwrite '%s'.\n", resolved_out_path); fclose(f_in); return 1; } #ifdef _WIN32 f_out = fopen(resolved_out_path, "wb"); #else // Restrict permissions to 0644 const int fd = open(resolved_out_path, O_CREAT | O_WRONLY | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); if (fd == -1) { zxc_log("Error creating output %s: %s\n", resolved_out_path, strerror(errno)); fclose(f_in); return 1; } f_out = fdopen(fd, "wb"); #endif if (!f_out) { zxc_log("Error open output %s: %s\n", resolved_out_path, strerror(errno)); if (f_in) fclose(f_in); #ifndef _WIN32 if (fd != -1) close(fd); #endif return 1; } created_out_file = 1; } // Prevent writing binary data to the terminal unless forced if (use_stdout && isatty(fileno(stdout)) && mode == MODE_COMPRESS && !force) { zxc_log( "Refusing to write compressed data to terminal.\n" "For help, type: zxc -h\n"); if (f_in) fclose(f_in); if (f_out) fclose(f_out); return 1; } // Set stdin/stdout to binary mode if using them #ifdef _WIN32 if (use_stdin) _setmode(_fileno(stdin), _O_BINARY); if (use_stdout) _setmode(_fileno(stdout), _O_BINARY); #else // On POSIX systems, there's no text/binary distinction, but we ensure // no buffering issues occur by using freopen if needed if (use_stdin) { if (!freopen(NULL, "rb", stdin)) zxc_log("Warning: Failed to reopen stdin in binary mode\n"); } if (use_stdout) { if (!freopen(NULL, "wb", stdout)) zxc_log("Warning: Failed to reopen stdout in binary mode\n"); } #endif // Determine if we should show progress bar and get file size // IMPORTANT: This must be done BEFORE setting large buffers with setvbuf // to avoid buffer inconsistency issues when reading the footer int show_progress = 0; uint64_t total_size = 0; if (!g_quiet && !use_stdout && !use_stdin && isatty(fileno(stderr))) { // Get file size based on mode if (mode == MODE_COMPRESS) { // Compression: get input file size const long long saved_pos = ftello(f_in); if (saved_pos >= 0) { if (fseeko(f_in, 0, SEEK_END) == 0) { const long long size = ftello(f_in); if (size > 0) total_size = (uint64_t)size; fseeko(f_in, saved_pos, SEEK_SET); } } } else { // Decompression: get decompressed size from footer (BEFORE starting decompression) const int64_t decomp_size = zxc_stream_get_decompressed_size(f_in); if (decomp_size > 0) total_size = (uint64_t)decomp_size; } // Only show progress for files > 1MB if (total_size > ZXC_IO_BUFFER_SIZE) show_progress = 1; } // Set large buffers for I/O performance (AFTER file size detection) char *b1 = malloc(ZXC_IO_BUFFER_SIZE), *b2 = malloc(ZXC_IO_BUFFER_SIZE); if (b1) setvbuf(f_in, b1, _IOFBF, ZXC_IO_BUFFER_SIZE); if (f_out && b2) setvbuf(f_out, b2, _IOFBF, ZXC_IO_BUFFER_SIZE); if (in_path && !g_quiet) { zxc_log_v("Processing %s... (Compression Level %d)\n", in_path, level); } if (g_verbose) zxc_log("Checksum: %s\n", checksum_enabled ? "enabled" : "disabled"); if (g_verbose && seekable) zxc_log("Seekable: enabled\n"); // Prepare progress context progress_ctx_t pctx = {.start_time = zxc_now(), .operation = (mode == MODE_COMPRESS) ? "Compressing" : "Decompressing", .total_size = total_size}; const double t0 = zxc_now(); int64_t bytes; if (mode == MODE_COMPRESS) { zxc_compress_opts_t copts = { .n_threads = num_threads, .level = level, .block_size = block_size, .checksum_enabled = checksum_enabled, .seekable = seekable, .progress_cb = show_progress ? cli_progress_callback : NULL, .user_data = &pctx, }; bytes = zxc_stream_compress(f_in, f_out, &copts); } else { zxc_decompress_opts_t dopts = { .n_threads = num_threads, .checksum_enabled = checksum_enabled, .progress_cb = show_progress ? cli_progress_callback : NULL, .user_data = &pctx, }; bytes = zxc_stream_decompress(f_in, f_out, &dopts); } const double dt = zxc_now() - t0; // Clear progress line on completion if (show_progress) { fprintf(stderr, "\r\033[K"); fflush(stderr); } if (!use_stdin) fclose(f_in); else setvbuf(stdin, NULL, _IONBF, 0); if (f_out && f_out != stdout) fclose(f_out); else if (use_stdout) { fflush(stdout); setvbuf(stdout, NULL, _IONBF, 0); } free(b1); free(b2); if (bytes >= 0) { if (mode == MODE_INTEGRITY) { // Test mode: show result if (json_output) { printf( "{\n" " \"filename\": \"%s\",\n" " \"status\": \"ok\",\n" " \"checksum_verified\": %s,\n" " \"time_seconds\": %.6f\n" "}\n", in_path ? in_path : "", checksum_enabled ? "true" : "false", dt); } else if (g_verbose) { printf( "%s: OK\n" " Checksum: %s\n" " Time: %.3fs\n", in_path ? in_path : "", checksum_enabled ? "verified (RapidHash)" : "not verified", dt); } else { printf("%s: OK\n", in_path ? in_path : ""); } } else { zxc_log_v("Processed %lld bytes in %.3fs\n", (long long)bytes, dt); } if (!use_stdin && !use_stdout && !keep_input && mode != MODE_INTEGRITY) unlink(resolved_in_path); } else { if (mode == MODE_INTEGRITY) { if (json_output) { printf( "{\n" " \"filename\": \"%s\",\n" " \"status\": \"failed\",\n" " \"error\": \"Integrity check failed (corrupted data or invalid checksum)\"\n" "}\n", in_path ? in_path : ""); } else { fprintf(stderr, "%s: FAILED\n", in_path ? in_path : ""); if (g_verbose) fprintf( stderr, " Reason: Integrity check failed (corrupted data or invalid checksum)\n"); } } else { zxc_log("Operation failed on %s.\n", in_path ? in_path : ""); if (created_out_file) unlink(resolved_out_path); } overall_ret = 1; } return overall_ret; } /** * @brief Main entry point. * Parses arguments and dispatches execution to Benchmark, Compress, or * Decompress modes. */ int main(int argc, char** argv) { zxc_mode_t mode = MODE_COMPRESS; int num_threads = 0; int keep_input = 0; int force = 0; int to_stdout = 0; int bench_seconds = 5; int checksum = -1; int level = 3; int json_output = 0; size_t block_size = 0; int seekable = 0; static const struct option long_options[] = {{"compress", no_argument, 0, 'z'}, {"decompress", no_argument, 0, 'd'}, {"list", no_argument, 0, 'l'}, {"test", no_argument, 0, 't'}, {"bench", optional_argument, 0, 'b'}, {"threads", required_argument, 0, 'T'}, {"keep", no_argument, 0, 'k'}, {"force", no_argument, 0, 'f'}, {"stdout", no_argument, 0, 'c'}, {"verbose", no_argument, 0, 'v'}, {"quiet", no_argument, 0, 'q'}, {"checksum", no_argument, 0, 'C'}, {"no-checksum", no_argument, 0, 'N'}, {"json", no_argument, 0, 'j'}, {"version", no_argument, 0, 'V'}, {"help", no_argument, 0, 'h'}, {"multiple", no_argument, 0, 'm'}, {"recursive", no_argument, 0, 'r'}, {"block-size", required_argument, 0, 'B'}, {"seekable", no_argument, 0, 'S'}, {0, 0, 0, 0}}; int opt; int multiple_mode = 0; int recursive_mode = 0; while ((opt = getopt_long(argc, argv, "123456b::B:cCdfhjklmrNqST:tvVz", long_options, NULL)) != -1) { switch (opt) { case 'z': mode = MODE_COMPRESS; break; case 'd': mode = MODE_DECOMPRESS; break; case 'l': mode = MODE_LIST; break; case 't': mode = MODE_INTEGRITY; break; case 'b': mode = MODE_BENCHMARK; const char* bench_arg = optarg; if (!bench_arg && optind < argc && argv[optind][0] >= '1' && argv[optind][0] <= '9') bench_arg = argv[optind++]; if (bench_arg) { bench_seconds = atoi(bench_arg); if (bench_seconds < 1 || bench_seconds > 3600) { fprintf(stderr, "Error: duration must be between 1 and 3600 seconds\n"); return 1; } } break; case '1': level = 1; break; case '2': level = 2; break; case '3': level = 3; break; case '4': level = 4; break; case '5': level = 5; break; case '6': level = 6; break; case 'T': num_threads = atoi(optarg); if (num_threads < 0 || num_threads > ZXC_MAX_THREADS) { fprintf(stderr, "Error: num_threads must be between 0 and %d\n", ZXC_MAX_THREADS); return 1; } break; case 'k': keep_input = 1; break; case 'f': force = 1; break; case 'c': to_stdout = 1; break; case 'v': g_verbose = 1; break; case 'q': g_quiet = 1; break; case 'C': checksum = 1; break; case 'N': checksum = 0; break; case 'j': json_output = 1; break; case 'm': multiple_mode = 1; break; case 'S': seekable = 1; break; case 'r': recursive_mode = 1; multiple_mode = 1; // Recursive implies multiple mode for files processing break; case 'B': { char* end = NULL; const long long bs_val = strtoll(optarg, &end, 10); if (bs_val <= 0 || end == optarg) { fprintf(stderr, "Error: block-size must be a power of 2 between 4K and 2M\n" " Examples: -B 4K, -B 128K, -B 1M, -B 2M\n"); return 1; } long long multiplier = 1; if (end && (*end == 'k' || *end == 'K')) { multiplier = 1024; end++; if (*end == 'b' || *end == 'B') end++; // optional "B" in "KB" } else if (end && (*end == 'm' || *end == 'M')) { multiplier = 1024 * 1024; end++; if (*end == 'b' || *end == 'B') end++; // optional "B" in "MB" } const long long bs_bytes = bs_val * multiplier; if (!zxc_validate_block_size((size_t)bs_bytes)) { fprintf(stderr, "Error: block-size must be a power of 2 between 4K and 2M\n" " Examples: -B 4K, -B 128K, -B 1M, -B 2M\n"); return 1; } block_size = (size_t)bs_bytes; break; } case '?': print_help(argv[0]); return 1; case 'V': print_version(); return 0; case 'h': print_help(argv[0]); return 0; default: return 1; } } // Handle positional arguments for mode selection (e.g., "zxc z file") if (optind < argc && mode != MODE_BENCHMARK) { if (strcmp(argv[optind], "z") == 0) { mode = MODE_COMPRESS; optind++; } else if (strcmp(argv[optind], "d") == 0) { mode = MODE_DECOMPRESS; optind++; } else if (strcmp(argv[optind], "l") == 0 || strcmp(argv[optind], "list") == 0) { mode = MODE_LIST; optind++; } else if (strcmp(argv[optind], "t") == 0 || strcmp(argv[optind], "test") == 0) { mode = MODE_INTEGRITY; optind++; } else if (strcmp(argv[optind], "b") == 0) { mode = MODE_BENCHMARK; optind++; } } if (checksum == -1) { checksum = (mode == MODE_BENCHMARK) ? 0 : 1; } /* * Benchmark Mode * Loads the entire input file into RAM to measure raw algorithm throughput * without disk I/O bottlenecks. */ if (mode == MODE_BENCHMARK) { if (optind >= argc) { zxc_log("Benchmark requires input file.\n"); return 1; } const char* in_path = argv[optind]; int ret = 1; uint8_t* ram = NULL; uint8_t* c_dat = NULL; FILE* fm = NULL; char resolved_path[4096]; if (zxc_validate_input_path(in_path, resolved_path, sizeof(resolved_path)) != 0) { zxc_log("Error: Invalid input file '%s': %s\n", in_path, strerror(errno)); return 1; } if (num_threads < 0 || num_threads > ZXC_MAX_THREADS) { zxc_log("Error: num_threads must be between 0 and %d\n", ZXC_MAX_THREADS); return 1; } FILE* f_in = fopen(resolved_path, "rb"); if (!f_in) goto bench_cleanup; if (fseeko(f_in, 0, SEEK_END) != 0) goto bench_cleanup; const long long fsize = ftello(f_in); if (fsize <= 0) goto bench_cleanup; const size_t in_size = (size_t)fsize; if (fseeko(f_in, 0, SEEK_SET) != 0) goto bench_cleanup; ram = malloc(in_size); if (!ram) goto bench_cleanup; if (fread(ram, 1, in_size, f_in) != in_size) goto bench_cleanup; fclose(f_in); f_in = NULL; if (!json_output) printf( "Input: %s (%zu bytes)\n" "Running for %d seconds (threads: %d)...\n", in_path, in_size, bench_seconds, num_threads); #ifdef _WIN32 if (!json_output) printf("Note: Using tmpfile on Windows (slower than fmemopen).\n"); fm = tmpfile(); if (fm) { fwrite(ram, 1, in_size, fm); rewind(fm); } #else fm = fmemopen(ram, in_size, "rb"); #endif if (!fm) goto bench_cleanup; double best_compress = 1e30; int compress_iters = 0; const double compress_deadline = zxc_now() + (double)bench_seconds; const double compress_start = zxc_now(); while (zxc_now() < compress_deadline) { rewind(fm); const double t0 = zxc_now(); zxc_compress_opts_t bench_copts = {.n_threads = num_threads, .level = level, .block_size = block_size, .checksum_enabled = checksum}; zxc_stream_compress(fm, NULL, &bench_copts); const double dt = zxc_now() - t0; if (dt < best_compress) best_compress = dt; compress_iters++; if (!json_output && !g_quiet) fprintf(stderr, "\rCompressing... %d iters (%.1fs)", compress_iters, zxc_now() - compress_start); } if (!json_output && !g_quiet) fprintf(stderr, "\r\033[K"); fclose(fm); fm = NULL; const uint64_t max_c = zxc_compress_bound(in_size); c_dat = malloc((size_t)max_c); if (!c_dat) goto bench_cleanup; #ifdef _WIN32 FILE* fm_in = tmpfile(); FILE* fm_out = tmpfile(); if (!fm_in || !fm_out) { if (fm_in) fclose(fm_in); if (fm_out) fclose(fm_out); goto bench_cleanup; } fwrite(ram, 1, in_size, fm_in); rewind(fm_in); #else FILE* fm_in = fmemopen(ram, in_size, "rb"); FILE* fm_out = fmemopen(c_dat, max_c, "wb"); if (!fm_in || !fm_out) { if (fm_in) fclose(fm_in); if (fm_out) fclose(fm_out); goto bench_cleanup; } #endif zxc_compress_opts_t bench_copts2 = {.n_threads = num_threads, .level = level, .block_size = block_size, .checksum_enabled = checksum}; const int64_t c_sz = zxc_stream_compress(fm_in, fm_out, &bench_copts2); if (c_sz < 0) { fclose(fm_in); fclose(fm_out); fm_in = NULL; fm_out = NULL; goto bench_cleanup; } #ifdef _WIN32 rewind(fm_out); fread(c_dat, 1, (size_t)c_sz, fm_out); fclose(fm_in); fclose(fm_out); #else fclose(fm_in); fclose(fm_out); #endif #ifdef _WIN32 FILE* fc = tmpfile(); if (!fc) goto bench_cleanup; fwrite(c_dat, 1, (size_t)c_sz, fc); rewind(fc); #else FILE* fc = fmemopen(c_dat, (size_t)c_sz, "rb"); if (!fc) goto bench_cleanup; #endif double best_decompress = 1e30; int decompress_iters = 0; const double decompress_deadline = zxc_now() + (double)bench_seconds; const double decompress_start = zxc_now(); while (zxc_now() < decompress_deadline) { rewind(fc); const double t0 = zxc_now(); zxc_decompress_opts_t bench_dopts = {.n_threads = num_threads, .checksum_enabled = checksum}; zxc_stream_decompress(fc, NULL, &bench_dopts); const double dt = zxc_now() - t0; if (dt < best_decompress) best_decompress = dt; decompress_iters++; if (!json_output && !g_quiet) fprintf(stderr, "\rDecompressing... %d iters (%.1fs)", decompress_iters, zxc_now() - decompress_start); } if (!json_output && !g_quiet) fprintf(stderr, "\r\033[K"); fclose(fc); const double compress_speed_mbps = (double)in_size / (1000.0 * 1000.0) / best_compress; const double decompress_speed_mbps = (double)in_size / (1000.0 * 1000.0) / best_decompress; const double ratio = (c_sz > 0) ? ((double)in_size / c_sz) : 0.0; if (json_output) printf( "{\n" " \"input_file\": \"%s\",\n" " \"input_size_bytes\": %zu,\n" " \"compressed_size_bytes\": %lld,\n" " \"compression_ratio\": %.3f,\n" " \"duration_seconds\": %d,\n" " \"compress_iterations\": %d,\n" " \"decompress_iterations\": %d,\n" " \"threads\": %d,\n" " \"level\": %d,\n" " \"checksum_enabled\": %s,\n" " \"compress_speed_mbps\": %.3f,\n" " \"decompress_speed_mbps\": %.3f,\n" " \"compress_time_seconds\": %.6f,\n" " \"decompress_time_seconds\": %.6f\n" "}\n", in_path, in_size, (long long)c_sz, ratio, bench_seconds, compress_iters, decompress_iters, num_threads, level, checksum ? "true" : "false", compress_speed_mbps, decompress_speed_mbps, best_compress, best_decompress); else printf( "Compressed: %lld bytes (ratio %.3f)\n" "Compress : %.3f MB/s (%d iters)\n" "Decompress: %.3f MB/s (%d iters)\n", (long long)c_sz, ratio, compress_speed_mbps, compress_iters, decompress_speed_mbps, decompress_iters); ret = 0; bench_cleanup: if (fm) fclose(fm); if (f_in) fclose(f_in); free(ram); free(c_dat); return ret; } /* * List Mode * Displays archive information (compressed size, uncompressed size, ratio). */ if (mode == MODE_LIST) { if (optind >= argc) { zxc_log("List mode requires input file.\n"); return 1; } int ret = 0; const int num_files = argc - optind; if (json_output && num_files > 1) printf("[\n"); for (int i = optind; i < argc; i++) { ret |= zxc_list_archive(argv[i], json_output); if (json_output && num_files > 1 && i < argc - 1) { printf(",\n"); } } if (json_output && num_files > 1) { printf("]\n"); } return ret; } if (multiple_mode && to_stdout) { zxc_log("Error: cannot write to stdout when using multiple files mode (-m).\n"); return 1; } /* * File Processing Mode * Loops over files and determines input/output paths. */ int overall_ret = 0; const int start_optind = optind; // If no files passed but we aren't using stdin, or mode expects files: if (optind >= argc && mode == MODE_INTEGRITY) { zxc_log("Test mode requires at least one input file.\n"); return 1; } if (multiple_mode && optind >= argc) { zxc_log("Multiple files mode requires at least one input file.\n"); return 1; } // Default to processing at least once (for stdin) if no files are passed and not in a mode that // strictly needs files const int num_files_to_process = (optind < argc) ? (argc - optind) : 1; for (int file_idx = 0; file_idx < num_files_to_process; file_idx++) { const char* current_arg = (optind < argc) ? argv[start_optind + file_idx] : NULL; if (recursive_mode && current_arg && strcmp(current_arg, "-") != 0 && zxc_is_directory(current_arg)) { overall_ret |= process_directory(current_arg, mode, num_threads, keep_input, force, to_stdout, checksum, level, block_size, json_output, seekable); } else { const char* explicit_out_path = (!multiple_mode && optind + 1 < argc && current_arg && strcmp(current_arg, "-") != 0 && !to_stdout) ? argv[start_optind + 1] : NULL; overall_ret |= process_single_file(current_arg, explicit_out_path, mode, num_threads, keep_input, force, to_stdout, checksum, level, block_size, json_output, seekable); } if (!multiple_mode) { break; // Standard mode only does the first argument as input } } return overall_ret; } zxc-0.11.0/src/lib/000077500000000000000000000000001520102567100137315ustar00rootroot00000000000000zxc-0.11.0/src/lib/vendors/000077500000000000000000000000001520102567100154115ustar00rootroot00000000000000zxc-0.11.0/src/lib/vendors/rapidhash.h000066400000000000000000000514151520102567100175330ustar00rootroot00000000000000/* * rapidhash V3 - Very fast, high quality, platform-independent hashing algorithm. * * Based on 'wyhash', by Wang Yi * * Copyright (C) 2025 Nicolas De Carli * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * * You can contact the author at: * - rapidhash source repository: https://github.com/Nicoshev/rapidhash */ #pragma once /* * Includes. */ #include #include #if defined(_MSC_VER) # include # if defined(_M_X64) && !defined(_M_ARM64EC) # pragma intrinsic(_umul128) # endif #endif /* * C/C++ macros. */ #ifdef _MSC_VER # define RAPIDHASH_ALWAYS_INLINE __forceinline #elif defined(__GNUC__) # define RAPIDHASH_ALWAYS_INLINE inline __attribute__((__always_inline__)) #else # define RAPIDHASH_ALWAYS_INLINE inline #endif #ifdef __cplusplus # define RAPIDHASH_NOEXCEPT noexcept # define RAPIDHASH_CONSTEXPR constexpr # ifndef RAPIDHASH_INLINE # define RAPIDHASH_INLINE RAPIDHASH_ALWAYS_INLINE # endif # if __cplusplus >= 201402L && !defined(_MSC_VER) # define RAPIDHASH_INLINE_CONSTEXPR RAPIDHASH_ALWAYS_INLINE constexpr # else # define RAPIDHASH_INLINE_CONSTEXPR RAPIDHASH_ALWAYS_INLINE # endif #else # define RAPIDHASH_NOEXCEPT # define RAPIDHASH_CONSTEXPR static const # ifndef RAPIDHASH_INLINE # define RAPIDHASH_INLINE static RAPIDHASH_ALWAYS_INLINE # endif # define RAPIDHASH_INLINE_CONSTEXPR RAPIDHASH_INLINE #endif /* * Unrolled macro. * Improves large input speed, but increases code size and worsens small input speed. * * RAPIDHASH_COMPACT: Normal behavior. * RAPIDHASH_UNROLLED: * */ #ifndef RAPIDHASH_UNROLLED # define RAPIDHASH_COMPACT #elif defined(RAPIDHASH_COMPACT) # error "cannot define RAPIDHASH_COMPACT and RAPIDHASH_UNROLLED simultaneously." #endif /* * Protection macro, alters behaviour of rapid_mum multiplication function. * * RAPIDHASH_FAST: Normal behavior, max speed. * RAPIDHASH_PROTECTED: Extra protection against entropy loss. */ #ifndef RAPIDHASH_PROTECTED # define RAPIDHASH_FAST #elif defined(RAPIDHASH_FAST) # error "cannot define RAPIDHASH_PROTECTED and RAPIDHASH_FAST simultaneously." #endif /* * Likely and unlikely macros. */ #if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__) # define _likely_(x) __builtin_expect(x,1) # define _unlikely_(x) __builtin_expect(x,0) #else # define _likely_(x) (x) # define _unlikely_(x) (x) #endif /* * Endianness macros. */ #ifndef RAPIDHASH_LITTLE_ENDIAN # if defined(_WIN32) || defined(__LITTLE_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) # define RAPIDHASH_LITTLE_ENDIAN # elif defined(__BIG_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) # define RAPIDHASH_BIG_ENDIAN # else # warning "could not determine endianness! Falling back to little endian." # define RAPIDHASH_LITTLE_ENDIAN # endif #endif /* * Default secret parameters. */ RAPIDHASH_CONSTEXPR uint64_t rapid_secret[8] = { 0x2d358dccaa6c78a5ull, 0x8bb84b93962eacc9ull, 0x4b33a62ed433d4a3ull, 0x4d5a2da51de1aa47ull, 0xa0761d6478bd642full, 0xe7037ed1a0b428dbull, 0x90ed1765281c388cull, 0xaaaaaaaaaaaaaaaaull}; /* * 64*64 -> 128bit multiply function. * * @param A Address of 64-bit number. * @param B Address of 64-bit number. * * Calculates 128-bit C = *A * *B. * * When RAPIDHASH_FAST is defined: * Overwrites A contents with C's low 64 bits. * Overwrites B contents with C's high 64 bits. * * When RAPIDHASH_PROTECTED is defined: * Xors and overwrites A contents with C's low 64 bits. * Xors and overwrites B contents with C's high 64 bits. */ RAPIDHASH_INLINE_CONSTEXPR void rapid_mum(uint64_t *A, uint64_t *B) RAPIDHASH_NOEXCEPT { #if defined(__SIZEOF_INT128__) __uint128_t r=*A; r*=*B; #ifdef RAPIDHASH_PROTECTED *A^=(uint64_t)r; *B^=(uint64_t)(r>>64); #else *A=(uint64_t)r; *B=(uint64_t)(r>>64); #endif #elif defined(_MSC_VER) && (defined(_WIN64) || defined(_M_HYBRID_CHPE_ARM64)) #if defined(_M_X64) #ifdef RAPIDHASH_PROTECTED uint64_t a, b; a=_umul128(*A,*B,&b); *A^=a; *B^=b; #else *A=_umul128(*A,*B,B); #endif #else #ifdef RAPIDHASH_PROTECTED uint64_t a, b; b = __umulh(*A, *B); a = *A * *B; *A^=a; *B^=b; #else uint64_t c = __umulh(*A, *B); *A = *A * *B; *B = c; #endif #endif #else uint64_t ha=*A>>32, hb=*B>>32, la=(uint32_t)*A, lb=(uint32_t)*B; uint64_t rh=ha*hb, rm0=ha*lb, rm1=hb*la, rl=la*lb, t=rl+(rm0<<32), c=t>32)+(rm1>>32)+c; #ifdef RAPIDHASH_PROTECTED *A^=lo; *B^=hi; #else *A=lo; *B=hi; #endif #endif } /* * Multiply and xor mix function. * * @param A 64-bit number. * @param B 64-bit number. * * Calculates 128-bit C = A * B. * Returns 64-bit xor between high and low 64 bits of C. */ RAPIDHASH_INLINE_CONSTEXPR uint64_t rapid_mix(uint64_t A, uint64_t B) RAPIDHASH_NOEXCEPT { rapid_mum(&A,&B); return A^B; } /* * Read functions. */ #ifdef RAPIDHASH_LITTLE_ENDIAN RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p, sizeof(uint64_t)); return v;} RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p, sizeof(uint32_t)); return v;} #elif defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__) RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p, sizeof(uint64_t)); return __builtin_bswap64(v);} RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p, sizeof(uint32_t)); return __builtin_bswap32(v);} #elif defined(_MSC_VER) RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p, sizeof(uint64_t)); return _byteswap_uint64(v);} RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p, sizeof(uint32_t)); return _byteswap_ulong(v);} #else RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p, 8); return (((v >> 56) & 0xff)| ((v >> 40) & 0xff00)| ((v >> 24) & 0xff0000)| ((v >> 8) & 0xff000000)| ((v << 8) & 0xff00000000)| ((v << 24) & 0xff0000000000)| ((v << 40) & 0xff000000000000)| ((v << 56) & 0xff00000000000000)); } RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p, 4); return (((v >> 24) & 0xff)| ((v >> 8) & 0xff00)| ((v << 8) & 0xff0000)| ((v << 24) & 0xff000000)); } #endif /* * rapidhash main function. * * @param key Buffer to be hashed. * @param len @key length, in bytes. * @param seed 64-bit seed used to alter the hash result predictably. * @param secret Triplet of 64-bit secrets used to alter hash result predictably. * * Returns a 64-bit hash. */ RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash_internal(const void *key, size_t len, uint64_t seed, const uint64_t* secret) RAPIDHASH_NOEXCEPT { const uint8_t *p=(const uint8_t *)key; seed ^= rapid_mix(seed ^ secret[2], secret[1]); uint64_t a=0, b=0; size_t i = len; if (_likely_(len <= 16)) { if (len >= 4) { seed ^= len; if (len >= 8) { const uint8_t* plast = p + len - 8; a = rapid_read64(p); b = rapid_read64(plast); } else { const uint8_t* plast = p + len - 4; a = rapid_read32(p); b = rapid_read32(plast); } } else if (len > 0) { a = (((uint64_t)p[0])<<45)|p[len-1]; b = p[len>>1]; } else a = b = 0; } else { if (len > 112) { uint64_t see1 = seed, see2 = seed; uint64_t see3 = seed, see4 = seed; uint64_t see5 = seed, see6 = seed; #ifdef RAPIDHASH_COMPACT do { seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); p += 112; i -= 112; } while(i > 112); #else while (i > 224) { seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); seed = rapid_mix(rapid_read64(p + 112) ^ secret[0], rapid_read64(p + 120) ^ seed); see1 = rapid_mix(rapid_read64(p + 128) ^ secret[1], rapid_read64(p + 136) ^ see1); see2 = rapid_mix(rapid_read64(p + 144) ^ secret[2], rapid_read64(p + 152) ^ see2); see3 = rapid_mix(rapid_read64(p + 160) ^ secret[3], rapid_read64(p + 168) ^ see3); see4 = rapid_mix(rapid_read64(p + 176) ^ secret[4], rapid_read64(p + 184) ^ see4); see5 = rapid_mix(rapid_read64(p + 192) ^ secret[5], rapid_read64(p + 200) ^ see5); see6 = rapid_mix(rapid_read64(p + 208) ^ secret[6], rapid_read64(p + 216) ^ see6); p += 224; i -= 224; } if (i > 112) { seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); p += 112; i -= 112; } #endif seed ^= see1; see2 ^= see3; see4 ^= see5; seed ^= see6; see2 ^= see4; seed ^= see2; } if (i > 16) { seed = rapid_mix(rapid_read64(p) ^ secret[2], rapid_read64(p + 8) ^ seed); if (i > 32) { seed = rapid_mix(rapid_read64(p + 16) ^ secret[2], rapid_read64(p + 24) ^ seed); if (i > 48) { seed = rapid_mix(rapid_read64(p + 32) ^ secret[1], rapid_read64(p + 40) ^ seed); if (i > 64) { seed = rapid_mix(rapid_read64(p + 48) ^ secret[1], rapid_read64(p + 56) ^ seed); if (i > 80) { seed = rapid_mix(rapid_read64(p + 64) ^ secret[2], rapid_read64(p + 72) ^ seed); if (i > 96) { seed = rapid_mix(rapid_read64(p + 80) ^ secret[1], rapid_read64(p + 88) ^ seed); } } } } } } a=rapid_read64(p+i-16) ^ i; b=rapid_read64(p+i-8); } a ^= secret[1]; b ^= seed; rapid_mum(&a, &b); return rapid_mix(a ^ secret[7], b ^ secret[1] ^ i); } /* * rapidhashMicro main function. * * @param key Buffer to be hashed. * @param len @key length, in bytes. * @param seed 64-bit seed used to alter the hash result predictably. * @param secret Triplet of 64-bit secrets used to alter hash result predictably. * * Returns a 64-bit hash. */ RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhashMicro_internal(const void *key, size_t len, uint64_t seed, const uint64_t* secret) RAPIDHASH_NOEXCEPT { const uint8_t *p=(const uint8_t *)key; seed ^= rapid_mix(seed ^ secret[2], secret[1]); uint64_t a=0, b=0; size_t i = len; if (_likely_(len <= 16)) { if (len >= 4) { seed ^= len; if (len >= 8) { const uint8_t* plast = p + len - 8; a = rapid_read64(p); b = rapid_read64(plast); } else { const uint8_t* plast = p + len - 4; a = rapid_read32(p); b = rapid_read32(plast); } } else if (len > 0) { a = (((uint64_t)p[0])<<45)|p[len-1]; b = p[len>>1]; } else a = b = 0; } else { if (i > 80) { uint64_t see1 = seed, see2 = seed; uint64_t see3 = seed, see4 = seed; do { seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); p += 80; i -= 80; } while(i > 80); seed ^= see1; see2 ^= see3; seed ^= see4; seed ^= see2; } if (i > 16) { seed = rapid_mix(rapid_read64(p) ^ secret[2], rapid_read64(p + 8) ^ seed); if (i > 32) { seed = rapid_mix(rapid_read64(p + 16) ^ secret[2], rapid_read64(p + 24) ^ seed); if (i > 48) { seed = rapid_mix(rapid_read64(p + 32) ^ secret[1], rapid_read64(p + 40) ^ seed); if (i > 64) { seed = rapid_mix(rapid_read64(p + 48) ^ secret[1], rapid_read64(p + 56) ^ seed); } } } } a=rapid_read64(p+i-16) ^ i; b=rapid_read64(p+i-8); } a ^= secret[1]; b ^= seed; rapid_mum(&a, &b); return rapid_mix(a ^ secret[7], b ^ secret[1] ^ i); } /* * rapidhashNano main function. * * @param key Buffer to be hashed. * @param len @key length, in bytes. * @param seed 64-bit seed used to alter the hash result predictably. * @param secret Triplet of 64-bit secrets used to alter hash result predictably. * * Returns a 64-bit hash. */ RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhashNano_internal(const void *key, size_t len, uint64_t seed, const uint64_t* secret) RAPIDHASH_NOEXCEPT { const uint8_t *p=(const uint8_t *)key; seed ^= rapid_mix(seed ^ secret[2], secret[1]); uint64_t a=0, b=0; size_t i = len; if (_likely_(len <= 16)) { if (len >= 4) { seed ^= len; if (len >= 8) { const uint8_t* plast = p + len - 8; a = rapid_read64(p); b = rapid_read64(plast); } else { const uint8_t* plast = p + len - 4; a = rapid_read32(p); b = rapid_read32(plast); } } else if (len > 0) { a = (((uint64_t)p[0])<<45)|p[len-1]; b = p[len>>1]; } else a = b = 0; } else { if (i > 48) { uint64_t see1 = seed, see2 = seed; do { seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); p += 48; i -= 48; } while(i > 48); seed ^= see1; seed ^= see2; } if (i > 16) { seed = rapid_mix(rapid_read64(p) ^ secret[2], rapid_read64(p + 8) ^ seed); if (i > 32) { seed = rapid_mix(rapid_read64(p + 16) ^ secret[2], rapid_read64(p + 24) ^ seed); } } a=rapid_read64(p+i-16) ^ i; b=rapid_read64(p+i-8); } a ^= secret[1]; b ^= seed; rapid_mum(&a, &b); return rapid_mix(a ^ secret[7], b ^ secret[1] ^ i); } /* * rapidhash seeded hash function. * * @param key Buffer to be hashed. * @param len @key length, in bytes. * @param seed 64-bit seed used to alter the hash result predictably. * * Calls rapidhash_internal using provided parameters and default secrets. * * Returns a 64-bit hash. */ RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash_withSeed(const void *key, size_t len, uint64_t seed) RAPIDHASH_NOEXCEPT { return rapidhash_internal(key, len, seed, rapid_secret); } /* * rapidhash general purpose hash function. * * @param key Buffer to be hashed. * @param len @key length, in bytes. * * Calls rapidhash_withSeed using provided parameters and the default seed. * * Returns a 64-bit hash. */ RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash(const void *key, size_t len) RAPIDHASH_NOEXCEPT { return rapidhash_withSeed(key, len, 0); } /* * rapidhashMicro seeded hash function. * * Designed for HPC and server applications, where cache misses make a noticeable performance detriment. * Clang-18+ compiles it to ~140 instructions without stack usage, both on x86-64 and aarch64. * Faster for sizes up to 512 bytes, just 15%-20% slower for inputs above 1kb. * * @param key Buffer to be hashed. * @param len @key length, in bytes. * @param seed 64-bit seed used to alter the hash result predictably. * * Calls rapidhash_internal using provided parameters and default secrets. * * Returns a 64-bit hash. */ RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhashMicro_withSeed(const void *key, size_t len, uint64_t seed) RAPIDHASH_NOEXCEPT { return rapidhashMicro_internal(key, len, seed, rapid_secret); } /* * rapidhashMicro hash function. * * @param key Buffer to be hashed. * @param len @key length, in bytes. * * Calls rapidhash_withSeed using provided parameters and the default seed. * * Returns a 64-bit hash. */ RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhashMicro(const void *key, size_t len) RAPIDHASH_NOEXCEPT { return rapidhashMicro_withSeed(key, len, 0); } /* * rapidhashNano seeded hash function. * * @param key Buffer to be hashed. * @param len @key length, in bytes. * @param seed 64-bit seed used to alter the hash result predictably. * * Calls rapidhash_internal using provided parameters and default secrets. * * Returns a 64-bit hash. */ RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhashNano_withSeed(const void *key, size_t len, uint64_t seed) RAPIDHASH_NOEXCEPT { return rapidhashNano_internal(key, len, seed, rapid_secret); } /* * rapidhashNano hash function. * * Designed for Mobile and embedded applications, where keeping a small code size is a top priority. * Clang-18+ compiles it to less than 100 instructions without stack usage, both on x86-64 and aarch64. * The fastest for sizes up to 48 bytes, but may be considerably slower for larger inputs. * * @param key Buffer to be hashed. * @param len @key length, in bytes. * * Calls rapidhash_withSeed using provided parameters and the default seed. * * Returns a 64-bit hash. */ RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhashNano(const void *key, size_t len) RAPIDHASH_NOEXCEPT { return rapidhashNano_withSeed(key, len, 0); } zxc-0.11.0/src/lib/zxc_common.c000066400000000000000000000711301520102567100162530ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /* * @file zxc_common.c * @brief Shared library utilities: context management, header I/O, bitpacking, * compress-bound calculation, and error-code name lookup. * * This translation unit contains the functions shared by both the buffer and * streaming APIs. It is linked into every build of libzxc. */ #include "../../include/zxc_buffer.h" #include "../../include/zxc_error.h" #include "../../include/zxc_sans_io.h" #include "zxc_internal.h" /* * ============================================================================ * CONTEXT MANAGEMENT * ============================================================================ */ /* * @brief Allocates memory aligned to the specified boundary. * * Uses `_aligned_malloc` on Windows and `posix_memalign` elsewhere. * * @param[in] size Number of bytes to allocate. * @param[in] alignment Required alignment (must be a power of two). * @return Pointer to the allocated block, or @c NULL on failure. */ void* zxc_aligned_malloc(const size_t size, const size_t alignment) { #if defined(_WIN32) return _aligned_malloc(size, alignment); #else void* ptr = NULL; if (posix_memalign(&ptr, alignment, size) != 0) return NULL; return ptr; #endif } /* * @brief Frees memory previously allocated by zxc_aligned_malloc(). * * @param[in] ptr Pointer returned by zxc_aligned_malloc() (may be @c NULL). */ void zxc_aligned_free(void* ptr) { #if defined(_WIN32) _aligned_free(ptr); #else free(ptr); #endif } /* * @brief Initialises a compression context, allocating all internal buffers. * * A single cache-line-aligned allocation is carved into hash table, chain * table, sequence buffers, token buffers, offset buffers, extra-length * buffers, and a literal buffer. * * @param[out] ctx Context to initialise (zeroed on entry). * @param[in] chunk_size Maximum uncompressed chunk size (bytes). * @param[in] mode 0 = decompression (skip buffer alloc), 1 = compression. * @param[in] level Compression level (stored in ctx). * @param[in] checksum_enabled Non-zero to enable checksum generation/verification. * @return @ref ZXC_OK on success, or @ref ZXC_ERROR_MEMORY on allocation failure. */ int zxc_cctx_init(zxc_cctx_t* RESTRICT ctx, const size_t chunk_size, const int mode, const int level, const int checksum_enabled) { ZXC_MEMSET(ctx, 0, sizeof(zxc_cctx_t)); ctx->checksum_enabled = checksum_enabled; /* Compute block-size derived parameters. */ ctx->chunk_size = chunk_size; const uint32_t offset_bits = zxc_log2_u32((uint32_t)chunk_size); ctx->offset_bits = offset_bits; ctx->offset_mask = (uint32_t)((1ULL << offset_bits) - 1); ctx->max_epoch = (uint32_t)(1ULL << (32 - offset_bits)); if (mode == 0) return ZXC_OK; const size_t max_seq = chunk_size / ZXC_LZ_MIN_MATCH_LEN + 16; const size_t sz_hash_pos = ZXC_LZ_HASH_SIZE * sizeof(uint32_t); const size_t sz_hash_tags = ZXC_LZ_HASH_SIZE * sizeof(uint8_t); const size_t sz_chain = ZXC_LZ_WINDOW_SIZE * sizeof(uint16_t); /* buf_sequences (GHI, level <= 2) aliases buf_offsets + buf_tokens (GLO, * level >= 3). Mutually exclusive per block; sized for the larger. */ const size_t sz_seq_union = max_seq * sizeof(uint32_t); /* Varint bytes per LL/ML: scales with chunk_size. */ const size_t vbyte_len = (offset_bits + 6) / 7; const size_t sz_extras = max_seq * 2 * vbyte_len; const size_t sz_lit = chunk_size + ZXC_PAD_SIZE; /* Calculate sizes with alignment padding (64 bytes for cache line alignment) */ size_t total_size = 0; const size_t off_hash_pos = total_size; total_size += ZXC_ALIGN_CL(sz_hash_pos); const size_t off_hash_tags = total_size; total_size += ZXC_ALIGN_CL(sz_hash_tags); const size_t off_chain = total_size; total_size += ZXC_ALIGN_CL(sz_chain); const size_t off_seq_union = total_size; total_size += ZXC_ALIGN_CL(sz_seq_union); const size_t off_extras = total_size; total_size += ZXC_ALIGN_CL(sz_extras); const size_t off_lit = total_size; total_size += ZXC_ALIGN_CL(sz_lit); uint8_t* const mem = (uint8_t*)zxc_aligned_malloc(total_size, ZXC_CACHE_LINE_SIZE); if (UNLIKELY(!mem)) return ZXC_ERROR_MEMORY; ctx->memory_block = mem; ctx->hash_table = (uint32_t*)(mem + off_hash_pos); ctx->hash_tags = (uint8_t*)(mem + off_hash_tags); ctx->chain_table = (uint16_t*)(mem + off_chain); ctx->buf_sequences = (uint32_t*)(mem + off_seq_union); ctx->buf_offsets = (uint16_t*)(mem + off_seq_union); ctx->buf_tokens = (uint8_t*)(mem + off_seq_union) + max_seq * sizeof(uint16_t); ctx->buf_extras = (uint8_t*)(mem + off_extras); ctx->literals = (uint8_t*)(mem + off_lit); ctx->compression_level = level; ctx->epoch = 1; ZXC_MEMSET(ctx->hash_table, 0, sz_hash_pos); ZXC_MEMSET(ctx->hash_tags, 0, sz_hash_tags); return ZXC_OK; } /* * @brief Releases all resources owned by a compression context. * * After this call every pointer inside @p ctx is @c NULL and the context * may be safely re-initialised with zxc_cctx_init(). * * @param[in,out] ctx Context to tear down. */ void zxc_cctx_free(zxc_cctx_t* ctx) { if (ctx->memory_block) { zxc_aligned_free(ctx->memory_block); ctx->memory_block = NULL; } if (ctx->lit_buffer) { free(ctx->lit_buffer); ctx->lit_buffer = NULL; } if (ctx->work_buf) { free(ctx->work_buf); ctx->work_buf = NULL; } if (ctx->opt_scratch) { zxc_aligned_free(ctx->opt_scratch); ctx->opt_scratch = NULL; } ctx->hash_table = NULL; ctx->hash_tags = NULL; ctx->chain_table = NULL; ctx->buf_sequences = NULL; ctx->buf_tokens = NULL; ctx->buf_offsets = NULL; ctx->buf_extras = NULL; ctx->literals = NULL; ctx->epoch = 0; ctx->lit_buffer_cap = 0; ctx->work_buf_cap = 0; ctx->opt_scratch_cap = 0; } /* * ============================================================================ * HEADER I/O * ============================================================================ */ /* * @brief Serialises a ZXC file header into @p dst. * * Layout (16 bytes): Magic (4) | Version (1) | Chunk (1) | Flags (1) | * Reserved (7) | CRC-16 (2). * * @param[out] dst Destination buffer (>= @ref ZXC_FILE_HEADER_SIZE bytes). * @param[in] dst_capacity Capacity of @p dst. * @param[in] has_checksum Non-zero to set the checksum flag. * @return Number of bytes written (@ref ZXC_FILE_HEADER_SIZE) on success, * or a negative @ref zxc_error_t code. */ int zxc_write_file_header(uint8_t* RESTRICT dst, const size_t dst_capacity, const size_t chunk_size, const int has_checksum) { if (UNLIKELY(dst_capacity < ZXC_FILE_HEADER_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; zxc_store_le32(dst, ZXC_MAGIC_WORD); dst[4] = ZXC_FILE_FORMAT_VERSION; // Block size stored as log2 exponent (e.g. 18 = 256 KB) dst[5] = (uint8_t)zxc_log2_u32((uint32_t)chunk_size); // Flags are at offset 6 dst[6] = has_checksum ? (ZXC_FILE_FLAG_HAS_CHECKSUM | ZXC_CHECKSUM_RAPIDHASH) : 0; // Bytes 7-13: Reserved (must be 0, 7 bytes) ZXC_MEMSET(dst + 7, 0, 7); // Bytes 14-15: CRC (16-bit) zxc_store_le16(dst + 14, 0); // Zero out before hashing const uint16_t crc = zxc_hash16(dst); zxc_store_le16(dst + 14, crc); return ZXC_FILE_HEADER_SIZE; } /* * @brief Parses and validates a ZXC file header from @p src. * * Checks the magic word, format version, and CRC-16. * * @param[in] src Source buffer (>= @ref ZXC_FILE_HEADER_SIZE bytes). * @param[in] src_size Size of @p src. * @param[out] out_block_size Receives the decoded block size (may be @c NULL). * @param[out] out_has_checksum Receives 1 if checksums are present, 0 otherwise * (may be @c NULL). * @return @ref ZXC_OK on success, or a negative @ref zxc_error_t code. */ int zxc_read_file_header(const uint8_t* RESTRICT src, const size_t src_size, size_t* RESTRICT out_block_size, int* RESTRICT out_has_checksum) { if (UNLIKELY(src_size < ZXC_FILE_HEADER_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; if (UNLIKELY(zxc_le32(src) != ZXC_MAGIC_WORD)) return ZXC_ERROR_BAD_MAGIC; if (UNLIKELY(src[4] != ZXC_FILE_FORMAT_VERSION)) return ZXC_ERROR_BAD_VERSION; uint8_t temp[ZXC_FILE_HEADER_SIZE]; ZXC_MEMCPY(temp, src, ZXC_FILE_HEADER_SIZE); // Zero out CRC bytes (14-15) before hash check temp[14] = 0; temp[15] = 0; if (UNLIKELY(zxc_le16(src + 14) != zxc_hash16(temp))) return ZXC_ERROR_BAD_HEADER; if (out_block_size) { const uint8_t code = src[5]; size_t block_size; if (LIKELY(code >= ZXC_BLOCK_SIZE_MIN_LOG2 && code <= ZXC_BLOCK_SIZE_MAX_LOG2)) { // Exponent encoding: block_size = 2^code (4 KB - 2 MB) block_size = (size_t)1U << code; } else if (code == 64) { // Legacy: hardcoded 256 KB default block_size = 256 * 1024; } else { return ZXC_ERROR_BAD_BLOCK_SIZE; } *out_block_size = block_size; } // Flags are at offset 6 if (out_has_checksum) *out_has_checksum = (src[6] & ZXC_FILE_FLAG_HAS_CHECKSUM) ? 1 : 0; return ZXC_OK; } /* * @brief Serialises a block header (8 bytes) into @p dst. * * @param[out] dst Destination buffer (>= @ref ZXC_BLOCK_HEADER_SIZE bytes). * @param[in] dst_capacity Capacity of @p dst. * @param[in] bh Populated block header descriptor. * @return Number of bytes written (@ref ZXC_BLOCK_HEADER_SIZE) on success, * or a negative @ref zxc_error_t code. */ int zxc_write_block_header(uint8_t* RESTRICT dst, const size_t dst_capacity, const zxc_block_header_t* RESTRICT bh) { if (UNLIKELY(dst_capacity < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; dst[0] = bh->block_type; dst[1] = 0; // Flags not used currently dst[2] = 0; // Reserved zxc_store_le32(dst + 3, bh->comp_size); dst[7] = 0; // Zero before hashing dst[7] = zxc_hash8(dst); // Checksum at the end return ZXC_BLOCK_HEADER_SIZE; } /* * @brief Parses and validates a block header from @p src. * * Validates the 8-bit CRC embedded in the header. * * @param[in] src Source buffer (>= @ref ZXC_BLOCK_HEADER_SIZE bytes). * @param[in] src_size Size of @p src. * @param[out] bh Receives the decoded block header fields. * @return @ref ZXC_OK on success, or a negative @ref zxc_error_t code. */ int zxc_read_block_header(const uint8_t* RESTRICT src, const size_t src_size, zxc_block_header_t* RESTRICT bh) { if (UNLIKELY(src_size < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; uint8_t temp[ZXC_BLOCK_HEADER_SIZE]; ZXC_MEMCPY(temp, src, ZXC_BLOCK_HEADER_SIZE); temp[7] = 0; // Zero out checksum byte before hashing if (UNLIKELY(src[7] != zxc_hash8(temp))) return ZXC_ERROR_BAD_HEADER; bh->block_type = src[0]; bh->block_flags = 0; // Flags not used currently bh->reserved = src[2]; bh->comp_size = zxc_le32(src + 3); bh->header_crc = src[7]; return ZXC_OK; } /* * @brief Writes the 12-byte file footer (source size + global checksum). * * @param[out] dst Destination buffer (>= @ref ZXC_FILE_FOOTER_SIZE bytes). * @param[in] dst_capacity Capacity of @p dst. * @param[in] src_size Original uncompressed size in bytes. * @param[in] global_hash Accumulated global checksum value. * @param[in] checksum_enabled Non-zero to write the checksum; zero to zero-fill. * @return Number of bytes written (@ref ZXC_FILE_FOOTER_SIZE) on success, * or a negative @ref zxc_error_t code. */ int zxc_write_file_footer(uint8_t* RESTRICT dst, const size_t dst_capacity, const uint64_t src_size, const uint32_t global_hash, const int checksum_enabled) { if (UNLIKELY(dst_capacity < ZXC_FILE_FOOTER_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; zxc_store_le64(dst, src_size); if (checksum_enabled) { zxc_store_le32(dst + sizeof(uint64_t), global_hash); } else { ZXC_MEMSET(dst + sizeof(uint64_t), 0, sizeof(uint32_t)); } return ZXC_FILE_FOOTER_SIZE; } /* * @brief Serialises a NUM block header (16 bytes). * * @param[out] dst Destination buffer (>= @ref ZXC_NUM_HEADER_BINARY_SIZE bytes). * @param[in] rem Remaining capacity of @p dst. * @param[in] nh Populated NUM header descriptor. * @return Number of bytes written on success, or a negative @ref zxc_error_t code. */ int zxc_write_num_header(uint8_t* RESTRICT dst, const size_t rem, const zxc_num_header_t* RESTRICT nh) { if (UNLIKELY(rem < ZXC_NUM_HEADER_BINARY_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; zxc_store_le64(dst, nh->n_values); zxc_store_le16(dst + 8, nh->frame_size); zxc_store_le16(dst + 10, 0); zxc_store_le32(dst + 12, 0); return ZXC_NUM_HEADER_BINARY_SIZE; } /* * @brief Parses a NUM block header from @p src. * * @param[in] src Source buffer (>= @ref ZXC_NUM_HEADER_BINARY_SIZE bytes). * @param[in] src_size Size of @p src. * @param[out] nh Receives the decoded NUM header fields. * @return @ref ZXC_OK on success, or a negative @ref zxc_error_t code. */ int zxc_read_num_header(const uint8_t* RESTRICT src, const size_t src_size, zxc_num_header_t* RESTRICT nh) { if (UNLIKELY(src_size < ZXC_NUM_HEADER_BINARY_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; nh->n_values = zxc_le64(src); nh->frame_size = zxc_le16(src + 8); return ZXC_OK; } /* * @brief Serialises a GLO block header followed by its section descriptors. * * @param[out] dst Destination buffer. * @param[in] rem Remaining capacity of @p dst. * @param[in] gh Populated GLO header descriptor. * @param[in] desc Array of @ref ZXC_GLO_SECTIONS section descriptors. * @return Total bytes written on success, or a negative @ref zxc_error_t code. */ int zxc_write_glo_header_and_desc(uint8_t* RESTRICT dst, const size_t rem, const zxc_gnr_header_t* RESTRICT gh, const zxc_section_desc_t desc[ZXC_GLO_SECTIONS]) { const size_t needed = ZXC_GLO_HEADER_BINARY_SIZE + ZXC_GLO_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; if (UNLIKELY(rem < needed)) return ZXC_ERROR_DST_TOO_SMALL; zxc_store_le32(dst, gh->n_sequences); zxc_store_le32(dst + 4, gh->n_literals); dst[8] = gh->enc_lit; dst[9] = gh->enc_litlen; dst[10] = gh->enc_mlen; dst[11] = gh->enc_off; zxc_store_le32(dst + 12, 0); uint8_t* p = dst + ZXC_GLO_HEADER_BINARY_SIZE; for (int i = 0; i < ZXC_GLO_SECTIONS; i++) { zxc_store_le64(p, desc[i].sizes); p += ZXC_SECTION_DESC_BINARY_SIZE; } return (int)needed; } /* * @brief Parses a GLO block header and its section descriptors from @p src. * * @param[in] src Source buffer. * @param[in] len Size of @p src. * @param[out] gh Receives the decoded GLO header. * @param[out] desc Receives @ref ZXC_GLO_SECTIONS decoded section descriptors. * @return @ref ZXC_OK on success, or a negative @ref zxc_error_t code. */ int zxc_read_glo_header_and_desc(const uint8_t* RESTRICT src, const size_t len, zxc_gnr_header_t* RESTRICT gh, zxc_section_desc_t desc[ZXC_GLO_SECTIONS]) { const size_t needed = ZXC_GLO_HEADER_BINARY_SIZE + ZXC_GLO_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; if (UNLIKELY(len < needed)) return ZXC_ERROR_SRC_TOO_SMALL; gh->n_sequences = zxc_le32(src); gh->n_literals = zxc_le32(src + 4); gh->enc_lit = src[8]; gh->enc_litlen = src[9]; gh->enc_mlen = src[10]; gh->enc_off = src[11]; const uint8_t* p = src + ZXC_GLO_HEADER_BINARY_SIZE; for (int i = 0; i < ZXC_GLO_SECTIONS; i++) { desc[i].sizes = zxc_le64(p); p += ZXC_SECTION_DESC_BINARY_SIZE; } return ZXC_OK; } /* * @brief Serialises a GHI block header followed by its section descriptors. * * @param[out] dst Destination buffer. * @param[in] rem Remaining capacity of @p dst. * @param[in] gh Populated GHI header descriptor. * @param[in] desc Array of @ref ZXC_GHI_SECTIONS section descriptors. * @return Total bytes written on success, or a negative @ref zxc_error_t code. */ int zxc_write_ghi_header_and_desc(uint8_t* RESTRICT dst, const size_t rem, const zxc_gnr_header_t* RESTRICT gh, const zxc_section_desc_t desc[ZXC_GHI_SECTIONS]) { const size_t needed = ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; if (UNLIKELY(rem < needed)) return ZXC_ERROR_DST_TOO_SMALL; zxc_store_le32(dst, gh->n_sequences); zxc_store_le32(dst + 4, gh->n_literals); dst[8] = gh->enc_lit; dst[9] = gh->enc_litlen; dst[10] = gh->enc_mlen; dst[11] = gh->enc_off; zxc_store_le32(dst + 12, 0); uint8_t* p = dst + ZXC_GHI_HEADER_BINARY_SIZE; for (int i = 0; i < ZXC_GHI_SECTIONS; i++) { zxc_store_le64(p, desc[i].sizes); p += ZXC_SECTION_DESC_BINARY_SIZE; } return (int)needed; } /* * @brief Parses a GHI block header and its section descriptors from @p src. * * @param[in] src Source buffer. * @param[in] len Size of @p src. * @param[out] gh Receives the decoded GHI header. * @param[out] desc Receives @ref ZXC_GHI_SECTIONS decoded section descriptors. * @return @ref ZXC_OK on success, or a negative @ref zxc_error_t code. */ int zxc_read_ghi_header_and_desc(const uint8_t* RESTRICT src, const size_t len, zxc_gnr_header_t* RESTRICT gh, zxc_section_desc_t desc[ZXC_GHI_SECTIONS]) { const size_t needed = ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; if (UNLIKELY(len < needed)) return ZXC_ERROR_SRC_TOO_SMALL; gh->n_sequences = zxc_le32(src); gh->n_literals = zxc_le32(src + 4); gh->enc_lit = src[8]; gh->enc_litlen = src[9]; gh->enc_mlen = src[10]; gh->enc_off = src[11]; const uint8_t* p = src + ZXC_GHI_HEADER_BINARY_SIZE; for (int i = 0; i < ZXC_GHI_SECTIONS; i++) { desc[i].sizes = zxc_le64(p); p += ZXC_SECTION_DESC_BINARY_SIZE; } return ZXC_OK; } /* * ============================================================================ * BITPACKING UTILITIES * ============================================================================ */ /* * @brief Bit-packs an array of 32-bit values into a compact byte stream. * * Each value is masked to @p bits width and packed contiguously. * * @param[in] src Source array of 32-bit integers. * @param[in] count Number of values to pack. * @param[out] dst Destination byte buffer. * @param[in] dst_cap Capacity of @p dst. * @param[in] bits Number of bits per value (0-32). * @return Number of bytes written on success, or a negative @ref zxc_error_t code. */ int zxc_bitpack_stream_32(const uint32_t* RESTRICT src, const size_t count, uint8_t* RESTRICT dst, const size_t dst_cap, const uint8_t bits) { const size_t out_bytes = ((count * bits) + CHAR_BIT - 1) / CHAR_BIT; // +4 bytes: packing may write past out_bytes when the last value straddles a byte boundary. const size_t safe_bytes = out_bytes + sizeof(uint32_t); if (UNLIKELY(dst_cap < safe_bytes)) return ZXC_ERROR_DST_TOO_SMALL; size_t bit_pos = 0; ZXC_MEMSET(dst, 0, safe_bytes); // Create a mask for the input bits to prevent overflow // If bits is 32, the shift (1ULL << 32) is undefined behavior on 32-bit types, // but here we use uint64_t. (1ULL << 32) is fine on 64-bit. // However, if bits=64 (unlikely for a 32-bit packer), it would be an issue. // For 0 < bits <= 32: const uint64_t val_mask = (bits == sizeof(uint32_t) * CHAR_BIT) ? UINT32_MAX : ((1ULL << bits) - 1); for (size_t i = 0; i < count; i++) { // Mask the input value to ensure we don't write garbage const uint64_t v = ((uint64_t)src[i] & val_mask) << (bit_pos % CHAR_BIT); const size_t byte_idx = bit_pos / CHAR_BIT; dst[byte_idx] |= (uint8_t)v; if (bits + (bit_pos % CHAR_BIT) > 1 * CHAR_BIT) dst[byte_idx + 1] |= (uint8_t)(v >> (1 * CHAR_BIT)); if (bits + (bit_pos % CHAR_BIT) > 2 * CHAR_BIT) dst[byte_idx + 2] |= (uint8_t)(v >> (2 * CHAR_BIT)); if (bits + (bit_pos % CHAR_BIT) > 3 * CHAR_BIT) dst[byte_idx + 3] |= (uint8_t)(v >> (3 * CHAR_BIT)); if (bits + (bit_pos % CHAR_BIT) > 4 * CHAR_BIT) dst[byte_idx + 4] |= (uint8_t)(v >> (4 * CHAR_BIT)); bit_pos += bits; } return (int)out_bytes; } /* * ============================================================================ * COMPRESS BOUND CALCULATION * ============================================================================ */ /* * @brief Returns the maximum compressed size for a given input size. * * The result accounts for the file header, per-block headers, block * checksums, worst-case expansion, EOF block, seekable overhead (SEK * block), and the file footer. * * The block count is derived from @ref ZXC_BLOCK_SIZE_MIN (4 KB) to * guarantee the bound holds for all valid block sizes and seekable mode. * * @param[in] input_size Uncompressed input size in bytes. * @return Upper bound on compressed size, or 0 if @p input_size would overflow. */ uint64_t zxc_compress_bound(const size_t input_size) { // Guard UINT64_MAX / SIZE_MAX would overflow. if (UNLIKELY(input_size > (SIZE_MAX - (SIZE_MAX >> 8)))) return 0; uint64_t n = ((uint64_t)input_size + ZXC_BLOCK_SIZE_MIN - 1) / ZXC_BLOCK_SIZE_MIN; if (n == 0) n = 1; return ZXC_FILE_HEADER_SIZE + (n * (ZXC_BLOCK_HEADER_SIZE + ZXC_BLOCK_CHECKSUM_SIZE + 64)) + (uint64_t)input_size + ZXC_BLOCK_HEADER_SIZE + /* EOF block */ ZXC_BLOCK_HEADER_SIZE + /* SEK block header (seekable) */ (n * ZXC_SEEK_ENTRY_SIZE) + /* SEK entries: 4 bytes per block */ ZXC_FILE_FOOTER_SIZE; } /* * @brief Returns the maximum compressed size for a single block (no file framing). * * @param[in] input_size Uncompressed block size in bytes. * @return Upper bound on compressed block size, or 0 on overflow. */ uint64_t zxc_compress_block_bound(const size_t input_size) { if (UNLIKELY(input_size > (SIZE_MAX - (SIZE_MAX >> 8)))) return 0; // Block header + worst-case expansion (64B overhead) + checksum return (uint64_t)ZXC_BLOCK_HEADER_SIZE + (uint64_t)input_size + 64 + ZXC_BLOCK_CHECKSUM_SIZE; } /* * @brief Returns the minimum dst_capacity required by zxc_decompress_block(). * * The decoder uses speculative wild-copy writes on its fast path. * Sizing the destination to uncompressed_size + ZXC_PAD_SIZE*66 guarantees * the fast path is always reachable and that tail bounds checks never * spuriously reject the last literals of a valid block. */ uint64_t zxc_decompress_block_bound(const size_t uncompressed_size) { if (UNLIKELY(uncompressed_size > SIZE_MAX - ZXC_DECOMPRESS_TAIL_PAD)) return 0; return (uint64_t)uncompressed_size + ZXC_DECOMPRESS_TAIL_PAD; } /* * @brief Estimates the total buffer bytes allocated inside a cctx for a block. * * Mirrors the persistent allocation layout in zxc_cctx_init(): each sub-buffer * is rounded up to the cache-line boundary, so the returned value matches the * single aligned allocation performed by the initializer. * * For @p level >= 6 the figure also includes ctx->opt_scratch (~8.125 bytes * per chunk_size byte: dp + parent_len + parent_off + a 1-bit-per-position * match-end bitmap), the cache-line-aligned scratch used by the optimal * parser. It is lazy-allocated on the first level-6 call and persists for * the lifetime of the cctx (no per-block malloc/free). */ uint64_t zxc_estimate_cctx_size(const size_t src_size, const int level) { if (UNLIKELY(src_size == 0)) return 0; const size_t chunk_size = zxc_block_size_ceil(src_size); const uint32_t offset_bits = zxc_log2_u32((uint32_t)chunk_size); const size_t max_seq = chunk_size / ZXC_LZ_MIN_MATCH_LEN + 16; const size_t vbyte_len = (offset_bits + 6) / 7; uint64_t total = 0; total += ZXC_ALIGN_CL(ZXC_LZ_HASH_SIZE * sizeof(uint32_t)); /* hash_table */ total += ZXC_ALIGN_CL(ZXC_LZ_HASH_SIZE * sizeof(uint8_t)); /* hash_tags */ total += ZXC_ALIGN_CL(ZXC_LZ_WINDOW_SIZE * sizeof(uint16_t)); /* chain_table (ring) */ /* sequences / tokens+offsets alias the same region (see zxc_cctx_init). */ total += ZXC_ALIGN_CL(max_seq * sizeof(uint32_t)); /* seq_union */ total += ZXC_ALIGN_CL(max_seq * 2 * vbyte_len); /* buf_extras */ total += ZXC_ALIGN_CL(chunk_size + ZXC_PAD_SIZE); /* literals */ /* The opaque wrapper struct allocated by zxc_create_cctx() adds a tiny * fixed overhead (< 128 B) that is negligible next to the per-chunk * buffers above and is intentionally omitted. */ if (level >= ZXC_LEVEL_DENSITY) { const size_t n_bm_words = (chunk_size + 1 + 63) / 64; size_t opt = ZXC_ALIGN_CL((chunk_size + 1) * sizeof(uint32_t)); /* dp */ opt += ZXC_ALIGN_CL((chunk_size + 1) * sizeof(uint16_t)); /* parent_len */ opt += ZXC_ALIGN_CL((chunk_size + 1) * sizeof(uint16_t)); /* parent_off */ opt += ZXC_ALIGN_CL(n_bm_words * sizeof(uint64_t)); /* match_end_bits */ /* opt_scratch is sized to hold both the DP arrays and (transiently) * the package-merge scratch for the Huffman code-length builder; * report the larger of the two. */ const size_t huf = ZXC_ALIGN_CL(ZXC_HUF_BUILD_SCRATCH_SIZE); total += (opt > huf) ? opt : huf; } return total; } /* * ============================================================================ * ERROR CODE UTILITIES * ============================================================================ */ /* * @brief Returns a human-readable string for the given error code. * * @param[in] code An error code from @ref zxc_error_t (or @ref ZXC_OK). * @return A static string such as @c "ZXC_OK" or @c "ZXC_ERROR_MEMORY". * Returns @c "ZXC_UNKNOWN_ERROR" for unrecognised codes. */ const char* zxc_error_name(const int code) { switch ((zxc_error_t)code) { case ZXC_OK: return "ZXC_OK"; case ZXC_ERROR_MEMORY: return "ZXC_ERROR_MEMORY"; case ZXC_ERROR_DST_TOO_SMALL: return "ZXC_ERROR_DST_TOO_SMALL"; case ZXC_ERROR_SRC_TOO_SMALL: return "ZXC_ERROR_SRC_TOO_SMALL"; case ZXC_ERROR_BAD_MAGIC: return "ZXC_ERROR_BAD_MAGIC"; case ZXC_ERROR_BAD_VERSION: return "ZXC_ERROR_BAD_VERSION"; case ZXC_ERROR_BAD_HEADER: return "ZXC_ERROR_BAD_HEADER"; case ZXC_ERROR_BAD_CHECKSUM: return "ZXC_ERROR_BAD_CHECKSUM"; case ZXC_ERROR_CORRUPT_DATA: return "ZXC_ERROR_CORRUPT_DATA"; case ZXC_ERROR_BAD_OFFSET: return "ZXC_ERROR_BAD_OFFSET"; case ZXC_ERROR_OVERFLOW: return "ZXC_ERROR_OVERFLOW"; case ZXC_ERROR_IO: return "ZXC_ERROR_IO"; case ZXC_ERROR_NULL_INPUT: return "ZXC_ERROR_NULL_INPUT"; case ZXC_ERROR_BAD_BLOCK_TYPE: return "ZXC_ERROR_BAD_BLOCK_TYPE"; case ZXC_ERROR_BAD_BLOCK_SIZE: return "ZXC_ERROR_BAD_BLOCK_SIZE"; default: return "ZXC_UNKNOWN_ERROR"; } } /* * ============================================================================ * LIBRARY INFORMATION * ============================================================================ */ /* * @brief Returns the minimum supported compression level. * * Returns the value of ZXC_LEVEL_FASTEST (currently 1). * This allows integrators to discover the level range at runtime without relying on * compile-time macros alone. */ int zxc_min_level(void) { return ZXC_LEVEL_FASTEST; } /* * @brief Returns the maximum supported compression level. * * Returns the value of ZXC_LEVEL_DENSITY (currently 6). */ int zxc_max_level(void) { return ZXC_LEVEL_DENSITY; } /* * @brief Returns the default compression level. * * Returns the value of ZXC_LEVEL_DEFAULT (currently 3). */ int zxc_default_level(void) { return ZXC_LEVEL_DEFAULT; } /* * @brief Returns the human-readable library version string. * * The returned pointer is a compile-time constant and must not be freed. * Example: "0.9.1". */ const char* zxc_version_string(void) { return ZXC_LIB_VERSION_STR; } zxc-0.11.0/src/lib/zxc_compress.c000066400000000000000000002640631520102567100166270ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /** * @file zxc_compress.c * @brief Block-level compression: LZ77 parsing, NUM / GLO / GHI / RAW encoding, * and the chunk-wrapper entry point. * * Compiled multiple times with different @c ZXC_FUNCTION_SUFFIX values to * produce AVX2, AVX-512, NEON, and scalar variants dispatched at runtime * by @ref zxc_dispatch.c. */ /* * Function Multi-Versioning Support * If ZXC_FUNCTION_SUFFIX is defined (e.g. _avx2, _neon), rename the public * entry point AND the Huffman entry points consumed by this TU. The defines * sit before zxc_internal.h so that the prototypes the header declares are * also rewritten with the suffix, keeping callers and callees consistent. */ #ifdef ZXC_FUNCTION_SUFFIX #define ZXC_CAT_IMPL(x, y) x##y #define ZXC_CAT(x, y) ZXC_CAT_IMPL(x, y) #define zxc_compress_chunk_wrapper ZXC_CAT(zxc_compress_chunk_wrapper, ZXC_FUNCTION_SUFFIX) #define zxc_huf_build_code_lengths ZXC_CAT(zxc_huf_build_code_lengths, ZXC_FUNCTION_SUFFIX) #define zxc_huf_encode_section ZXC_CAT(zxc_huf_encode_section, ZXC_FUNCTION_SUFFIX) #endif #include "../../include/zxc_error.h" #include "../../include/zxc_sans_io.h" #include "zxc_internal.h" /** * @brief Computes a hash value for either a 4-byte or 5-byte sequence. * * @param[in] val The 64-bit integer sequence (e.g., 8 bytes read from input stream). * @param[in] use_hash5 Non-zero to use the 5-byte xorshift64* hash (Marsaglia/Vigna), zero for * 4-byte Marsaglia hash. * @return uint32_t A hash value suitable for indexing the match table. */ static ZXC_ALWAYS_INLINE uint32_t zxc_hash_func(const uint64_t val, const int use_hash5) { if (use_hash5) { const uint64_t v5 = val & 0xFFFFFFFFFFULL; return (uint32_t)((v5 * ZXC_LZ_HASH_PRIME2) >> (64 - ZXC_LZ_HASH_BITS)); } else { const uint64_t v4 = val ^ (val >> 15); return ((uint32_t)v4 * ZXC_LZ_HASH_PRIME1) >> (32 - ZXC_LZ_HASH_BITS); } } #if defined(ZXC_USE_AVX2) /** * @brief Reduces a 256-bit integer vector to a single scalar by finding the maximum unsigned 32-bit * integer element. * * This function performs a horizontal reduction across the 8 packed 32-bit unsigned integers * in the source vector to determine the maximum value. * * @param[in] v The 256-bit vector containing 8 unsigned 32-bit integers. * @return The maximum unsigned 32-bit integer found in the vector. */ // codeql[cpp/unused-static-function] : Used conditionally when ZXC_USE_AVX2 is defined static ZXC_ALWAYS_INLINE uint32_t zxc_mm256_reduce_max_epu32(__m256i v) { __m128i vlow = _mm256_castsi256_si128(v); // Extract the lower 128 bits __m128i vhigh = _mm256_extracti128_si256(v, 1); // Extract the upper 128 bits vlow = _mm_max_epu32(vlow, vhigh); // Element-wise max of lower and upper halves __m128i vshuf = _mm_shuffle_epi32(vlow, _MM_SHUFFLE(1, 0, 3, 2)); // Shuffle to swap pairs vlow = _mm_max_epu32(vlow, vshuf); // Max of original and swapped vshuf = _mm_shuffle_epi32(vlow, _MM_SHUFFLE(2, 3, 0, 1)); // Shuffle to bring remaining candidates vlow = _mm_max_epu32(vlow, vshuf); // Final max comparison return (uint32_t)_mm_cvtsi128_si32(vlow); // Extract the scalar result } #endif /** * @brief Writes a Prefix Varint encoded value to a buffer. * * This function encodes a 32-bit unsigned integer using Prefix Varint encoding * and writes it to the destination buffer. Unary prefix bits in the first * byte determine the total length (1-5 bytes), allowing for branchless or * predictable decoding. * * Format: * - 0xxxxxxx (1 byte) * - 10xxxxxx ... (2 bytes) * - 110xxxxx ... (3 bytes) * ... * * @param[out] dst Pointer to the destination buffer where the encoded value will be written. * @param[in] val The 32-bit unsigned integer value to encode. * @return The number of bytes written to the destination buffer. */ static ZXC_ALWAYS_INLINE size_t zxc_write_varint(uint8_t* RESTRICT dst, const uint32_t val) { // 1 byte: 0xxxxxxx (7 bits) = 2^7 = 128 if (LIKELY(val < (1U << 7))) { dst[0] = (uint8_t)val; return 1; } // 2 bytes: 10xxxxxx xxxxxxxx (14 bits) = 2^14 = 16384 if (LIKELY(val < (1U << 14))) { dst[0] = (uint8_t)(0x80 | (val & 0x3F)); dst[1] = (uint8_t)(val >> 6); return 2; } // 3 bytes: 110xxxxx xxxxxxxx xxxxxxxx (21 bits) = 2^21 = 2097152 if (LIKELY(val < (1U << 21))) { dst[0] = (uint8_t)(0xC0 | (val & 0x1F)); dst[1] = (uint8_t)(val >> 5); dst[2] = (uint8_t)(val >> 13); return 3; } // 4 bytes: 1110xxxx xxxxxxxx xxxxxxxx xxxxxxxx (28 bits) = 2^28 = 268435456 if (val < (1U << 28)) { dst[0] = (uint8_t)(0xE0 | (val & 0x0F)); dst[1] = (uint8_t)(val >> 4); dst[2] = (uint8_t)(val >> 12); dst[3] = (uint8_t)(val >> 20); return 4; } // 5 bytes: 11110xxx xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx (32 bits) dst[0] = (uint8_t)(0xF0 | (val & 0x07)); dst[1] = (uint8_t)(val >> 3); dst[2] = (uint8_t)(val >> 11); dst[3] = (uint8_t)(val >> 19); dst[4] = (uint8_t)(val >> 27); return 5; } /** * @brief Structure representing a match found during compression. * * This structure holds information about a matching sequence found * in the input data during the compression process. * * @param ref Pointer to the reference data where the match was found. * @param len Length of the matching sequence in bytes. * @param backtrack Distance to backtrack from the current position to find the match. */ typedef struct { const uint8_t* ref; uint32_t len; uint32_t backtrack; } zxc_match_t; /** * @brief Finds the best matching sequence for LZ77 compression * * Uses a split hash table layout: * - hash_table[h] : uint32_t position + epoch (128 KB for 15-bit hash) * - hash_tags[h] : uint8_t tag for fast rejection (32 KB, L1-resident) * * @param[in] src Pointer to the start of the source buffer. * @param[in] ip Current input position pointer. * @param[in] iend Pointer to the end of the input buffer. * @param[in] mflimit Pointer to the match finding limit. * @param[in] anchor Pointer to the current anchor position. * @param[in,out] hash_table Pointer to the position table for match finding. * @param[in,out] hash_tags Pointer to the tag table for fast rejection. * @param[in,out] chain_table Pointer to the chain table for collision handling. * @param[in] epoch_mark Current epoch marker for hash table invalidation. * @param[in] p LZ77 parameters controlling search depth, lazy matching, and stepping. * @return zxc_match_t Structure containing the best match information * (reference pointer, length of the match, and backtrack distance). */ static ZXC_ALWAYS_INLINE zxc_match_t zxc_lz77_find_best_match( const uint8_t* src, const uint8_t* ip, const uint8_t* iend, const uint8_t* mflimit, const uint8_t* anchor, uint32_t* RESTRICT hash_table, uint8_t* RESTRICT hash_tags, uint16_t* RESTRICT chain_table, const uint32_t epoch_mark, const uint32_t offset_mask, const int level, const zxc_lz77_params_t p) { const int use_hash5 = (level >= 3); // Track the best match found so far. // ref is the pointer to the start of the match in the history buffer, // len is the match length, and backtrack is the distance from ip to ref. // Start with a sentinel length just below the minimum so any valid match will replace it. zxc_match_t best = (zxc_match_t){NULL, ZXC_LZ_MIN_MATCH_LEN - 1, 0}; // Load the 8-byte sequence at the current position. uint64_t cur_val8 = zxc_le64(ip); uint32_t cur_val = (uint32_t)cur_val8; uint32_t h = zxc_hash_func(cur_val8, use_hash5); // 8-bit tag: XOR fold of first 4 bytes for fast rejection const uint8_t cur_tag = (uint8_t)(cur_val ^ (cur_val >> 16)); // Current position in the input buffer expressed as a 32-bit index. const uint32_t cur_pos = (uint32_t)(ip - src); // Tag-first filter on fast levels. const uint8_t stored_tag = hash_tags[h]; uint32_t match_idx; if (level <= ZXC_LEVEL_FAST && stored_tag != cur_tag) { match_idx = 0; } else { const uint32_t raw_head = hash_table[h]; match_idx = ((raw_head & ~offset_mask) == epoch_mark) ? (raw_head & offset_mask) : 0; } // skip_head still drives the chain walk on level >= 3 (advances past the // mismatched head without comparing). On level <= 2 it is always 0 here: // either match_idx == 0 (filter-skip) or stored_tag == cur_tag. const int skip_head = (match_idx != 0) & (stored_tag != cur_tag); // Split table writes hash_table[h] = epoch_mark | cur_pos; hash_tags[h] = cur_tag; // Branchless chain table update const uint32_t dist = cur_pos - match_idx; const uint32_t valid_mask = -((int32_t)((match_idx != 0) & (dist < ZXC_LZ_WINDOW_SIZE))); chain_table[cur_pos & ZXC_LZ_WINDOW_MASK] = (uint16_t)(dist & valid_mask); if (match_idx == 0) return best; int attempts = p.search_depth; // Optimization: If head tag doesn't match, advance immediately without loading the first // mismatch. if (skip_head) { const uint16_t delta = chain_table[match_idx & ZXC_LZ_WINDOW_MASK]; const uint32_t next_idx = match_idx - delta; match_idx = (delta != 0) ? next_idx : 0; attempts--; } while (match_idx > 0 && attempts-- >= 0) { if (UNLIKELY(cur_pos - match_idx > ZXC_LZ_MAX_DIST)) break; const uint8_t* ref = src + match_idx; const uint32_t ref_val = zxc_le32(ref); const int tag_match = (ref_val == cur_val); // Simplified check: only tag match and next-byte match required const int should_compare = tag_match && (ref[best.len] == ip[best.len]); if (should_compare) { uint32_t mlen = sizeof(uint32_t); // We already know the first 4 bytes match // Fast path: Scalar 64-bit comparison for short matches (=< 64 bytes) // Most matches are short, so this avoids SIMD overhead for common cases const uint8_t* limit_8 = iend - sizeof(uint64_t); const uint8_t* scalar_limit = ip + mlen + 64; if (scalar_limit > limit_8) scalar_limit = limit_8; while (ip + mlen < scalar_limit) { uint64_t diff = zxc_le64(ip + mlen) ^ zxc_le64(ref + mlen); if (diff == 0) mlen += sizeof(uint64_t); else { mlen += (zxc_ctz64(diff) >> 3); goto _match_len_done; } } // Long match path: Use SIMD for matches exceeding 64 bytes #if defined(ZXC_USE_AVX512) const uint8_t* limit_64 = iend - 64; while (ip + mlen < limit_64) { const __m512i v_src = _mm512_loadu_si512((const void*)(ip + mlen)); const __m512i v_ref = _mm512_loadu_si512((const void*)(ref + mlen)); const __mmask64 mask = _mm512_cmpeq_epi8_mask(v_src, v_ref); if (mask == 0xFFFFFFFFFFFFFFFF) mlen += 64; else { mlen += (uint32_t)zxc_ctz64(~mask); goto _match_len_done; } } #elif defined(ZXC_USE_AVX2) const uint8_t* limit_32 = iend - 32; while (ip + mlen < limit_32) { const __m256i v_src = _mm256_loadu_si256((const __m256i*)(ip + mlen)); const __m256i v_ref = _mm256_loadu_si256((const __m256i*)(ref + mlen)); const __m256i v_cmp = _mm256_cmpeq_epi8(v_src, v_ref); const uint32_t mask = (uint32_t)_mm256_movemask_epi8(v_cmp); if (mask == 0xFFFFFFFF) mlen += 32; else { mlen += zxc_ctz32(~mask); goto _match_len_done; } } #elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) const uint8_t* limit_16 = iend - 16; while (ip + mlen < limit_16) { const uint8x16_t v_src = vld1q_u8(ip + mlen); const uint8x16_t v_ref = vld1q_u8(ref + mlen); const uint8x16_t v_cmp = vceqq_u8(v_src, v_ref); #if defined(ZXC_USE_NEON64) /* Compress 128-bit byte-mask -> 64-bit nibble-mask via * SHRN: each 0x00/0xFF byte becomes a 0x0/0xF nibble. */ const uint64_t mask = vget_lane_u64( vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(v_cmp), 4)), 0); if (LIKELY(mask == ~(uint64_t)0)) { mlen += 16; } else { mlen += (uint32_t)(zxc_ctz64(~mask) >> 2); goto _match_len_done; } #else uint8x8_t p1 = vpmin_u8(vget_low_u8(v_cmp), vget_high_u8(v_cmp)); uint8x8_t p2 = vpmin_u8(p1, p1); uint8x8_t p3 = vpmin_u8(p2, p2); uint8x8_t p4 = vpmin_u8(p3, p3); uint8_t min_val = vget_lane_u8(p4, 0); if (min_val == 0xFF) mlen += 16; else { uint8x16_t v_diff = vmvnq_u8(v_cmp); uint64_t lo = (uint64_t)vgetq_lane_u32(vreinterpretq_u32_u8(v_diff), 0) | ((uint64_t)vgetq_lane_u32(vreinterpretq_u32_u8(v_diff), 1) << 32); if (lo != 0) mlen += (zxc_ctz64(lo) >> 3); else mlen += 8 + (zxc_ctz64((uint64_t)vgetq_lane_u32(vreinterpretq_u32_u8(v_diff), 2) | ((uint64_t)vgetq_lane_u32(vreinterpretq_u32_u8(v_diff), 3) << 32)) >> 3); goto _match_len_done; } #endif } #endif while (ip + mlen < limit_8) { const uint64_t diff = zxc_le64(ip + mlen) ^ zxc_le64(ref + mlen); if (diff == 0) mlen += sizeof(uint64_t); else { mlen += (zxc_ctz64(diff) >> 3); goto _match_len_done; } } while (ip + mlen < iend && ref[mlen] == ip[mlen]) mlen++; _match_len_done:; const int better = (mlen > best.len); best.len = better ? mlen : best.len; best.ref = better ? ref : best.ref; if (UNLIKELY(best.len >= (uint32_t)p.sufficient_len || ip + best.len >= iend)) break; } const uint16_t delta = chain_table[match_idx & ZXC_LZ_WINDOW_MASK]; const uint32_t next_idx = match_idx - delta; ZXC_PREFETCH_READ(src + next_idx); match_idx = (delta != 0) ? next_idx : 0; } if (best.ref) { // Backtrack to extend match backwards const uint8_t* b_ip = ip; const uint8_t* b_ref = best.ref; while (b_ip > anchor && b_ref > src && b_ip[-1] == b_ref[-1]) { b_ip--; b_ref--; best.len++; best.backtrack++; } best.ref = b_ref; } if (p.use_lazy && best.ref && best.len < (uint32_t)p.lazy_len_threshold && ip + 1 < mflimit) { // --- Lazy evaluation at ip+1 --- const uint64_t next_val8 = zxc_le64(ip + 1); const uint32_t next_val = (uint32_t)next_val8; const uint32_t h2 = zxc_hash_func(next_val8, use_hash5); const uint8_t next_stored_tag = hash_tags[h2]; const uint32_t next_head = hash_table[h2]; uint32_t next_idx = (next_head & ~offset_mask) == epoch_mark ? (next_head & offset_mask) : 0; const uint8_t next_tag = (uint8_t)(next_val ^ (next_val >> 16)); const int skip_lazy_head = (next_idx > 0 && next_stored_tag != next_tag); uint32_t max_lazy2 = 0; int lazy_att = p.lazy_attempts; int is_lazy_first = 1; while (next_idx > 0 && lazy_att-- > 0) { if (UNLIKELY((uint32_t)(ip + 1 - src) - next_idx > ZXC_LZ_MAX_DIST)) break; const uint8_t* ref2 = src + next_idx; if ((!is_lazy_first || !skip_lazy_head) && zxc_le32(ref2) == next_val) { uint32_t l2 = sizeof(uint32_t); const uint8_t* limit = iend - sizeof(uint64_t); while (ip + 1 + l2 < limit) { const uint64_t v1 = zxc_le64(ip + 1 + l2); const uint64_t v2 = zxc_le64(ref2 + l2); if (v1 != v2) { l2 += (uint32_t)(zxc_ctz64(v1 ^ v2) >> 3); goto lazy2_done; } l2 += sizeof(uint64_t); } while (ip + 1 + l2 < iend && ref2[l2] == ip[1 + l2]) l2++; lazy2_done: max_lazy2 = l2 > max_lazy2 ? l2 : max_lazy2; } const uint16_t delta = chain_table[next_idx & ZXC_LZ_WINDOW_MASK]; if (UNLIKELY(delta == 0)) break; next_idx -= delta; is_lazy_first = 0; } // --- Lazy evaluation at ip+2 (computed in parallel, no dependency on lazy 1) --- uint32_t max_lazy3 = 0; if (level >= ZXC_LEVEL_BALANCED && ip + 2 < mflimit) { const uint64_t val3_8 = zxc_le64(ip + 2); const uint32_t val3 = (uint32_t)val3_8; const uint32_t h3 = zxc_hash_func(val3_8, use_hash5); const uint8_t tag3 = hash_tags[h3]; const uint32_t head3 = hash_table[h3]; uint32_t idx3 = (head3 & ~offset_mask) == epoch_mark ? (head3 & offset_mask) : 0; const uint8_t cur_tag3 = (uint8_t)(val3 ^ (val3 >> 16)); const int skip_head3 = (idx3 > 0 && tag3 != cur_tag3); int is_first3 = 1; lazy_att = p.lazy_attempts; while (idx3 > 0 && lazy_att-- > 0) { if (UNLIKELY((uint32_t)(ip + 2 - src) - idx3 > ZXC_LZ_MAX_DIST)) break; const uint8_t* ref3 = src + idx3; if ((!is_first3 || !skip_head3) && zxc_le32(ref3) == val3) { uint32_t l3 = sizeof(uint32_t); const uint8_t* limit = iend - sizeof(uint64_t); while (ip + 2 + l3 < limit) { const uint64_t v1 = zxc_le64(ip + 2 + l3); const uint64_t v2 = zxc_le64(ref3 + l3); if (v1 != v2) { l3 += (uint32_t)(zxc_ctz64(v1 ^ v2) >> 3); goto lazy3_done; } l3 += sizeof(uint64_t); } while (ip + 2 + l3 < iend && ref3[l3] == ip[2 + l3]) l3++; lazy3_done: max_lazy3 = l3 > max_lazy3 ? l3 : max_lazy3; } const uint16_t delta = chain_table[idx3 & ZXC_LZ_WINDOW_MASK]; if (UNLIKELY(delta == 0)) break; idx3 -= delta; is_first3 = 0; } } // Single decision: invalidate if either lazy position found a better match if (max_lazy2 > best.len + 1 || max_lazy3 > best.len + 2) best.ref = NULL; } return best; } /** * @brief Encodes a block of numerical data using delta encoding and * bit-packing. * * This function compresses a source buffer of 32-bit integers. It processes the * data in frames defined by `ZXC_NUM_FRAME_SIZE`. * * **Algorithm Steps:** * 1. **Delta Encoding:** Calculates `delta = value[i] - value[i-1]`. This * reduces the magnitude of numbers if the data is sequential or correlated. * - **SIMD Optimization:** Uses AVX2 (`_mm256_sub_epi32`) to compute deltas * for 8 integers at once. * 2. **ZigZag Encoding:** Maps signed deltas to unsigned integers (`(n << 1) ^ * (n >> 31)`). This ensures small negative numbers become small positive * numbers (e.g., -1 -> 1, 1 -> 2). * 3. **Bit Width Calculation:** Finds the maximum value in the frame to * determine the minimum number of bits (`b`) needed to represent all deltas. * 4. **Bit Packing:** Packs the ZigZag-encoded deltas into a compact bitstream * using `b` bits per value. * * @param[in] src Pointer to the source buffer containing raw 32-bit integer data. * @param[in] src_sz Size of the source buffer in bytes. Must be a multiple of 4 * and non-zero. * @param[out] dst Pointer to the destination buffer where compressed data will be * written. * @param[in] dst_cap Capacity of the destination buffer in bytes. * @param[out] out_sz Pointer to a variable where the total size of the compressed * output will be stored. * * @return ZXC_OK on success, or a negative zxc_error_t code (e.g., ZXC_ERROR_DST_TOO_SMALL) if an * error occurs (e.g., invalid input size, destination buffer too small). */ static int zxc_encode_block_num(const zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, size_t dst_cap, size_t* RESTRICT out_sz) { if (UNLIKELY(src_sz % sizeof(uint32_t) != 0 || src_sz == 0 || dst_cap < ZXC_BLOCK_HEADER_SIZE + ZXC_NUM_HEADER_BINARY_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; const size_t count = src_sz / sizeof(uint32_t); zxc_block_header_t bh = {.block_type = ZXC_BLOCK_NUM}; uint8_t* p_curr = dst + ZXC_BLOCK_HEADER_SIZE; size_t rem = dst_cap - ZXC_BLOCK_HEADER_SIZE; const zxc_num_header_t nh = {.n_values = count, .frame_size = ZXC_NUM_FRAME_SIZE}; const int hs = zxc_write_num_header(p_curr, rem, &nh); if (UNLIKELY(hs < 0)) return hs; p_curr += hs; rem -= hs; uint32_t deltas[ZXC_NUM_FRAME_SIZE]; const uint8_t* in_ptr = src; uint32_t prev = 0; for (size_t i = 0; i < count; i += ZXC_NUM_FRAME_SIZE) { const size_t frames = (count - i < ZXC_NUM_FRAME_SIZE) ? (count - i) : ZXC_NUM_FRAME_SIZE; uint32_t max_d = 0; const uint32_t base = prev; size_t j = 0; #if defined(ZXC_USE_AVX512) if (frames >= 16) { __m512i v_max_accum = _mm512_setzero_si512(); // Initialize max accumulator to 0 for (; j < (frames & ~15); j += 16) { if (UNLIKELY(i == 0 && j == 0)) goto _scalar; // Load 16 consecutive integers const __m512i vc = _mm512_loadu_si512((const void*)(in_ptr + j * 4)); // Load 16 integers offset by -1 to get previous values const __m512i vp = _mm512_loadu_si512((const void*)(in_ptr + j * 4 - 4)); const __m512i diff = _mm512_sub_epi32(vc, vp); // Compute deltas: curr - prev // ZigZag encode: (diff << 1) ^ (diff >> 31) const __m512i zigzag = _mm512_xor_si512(_mm512_slli_epi32(diff, 1), _mm512_srai_epi32(diff, 31)); _mm512_storeu_si512((void*)&deltas[j], zigzag); // Store results v_max_accum = _mm512_max_epu32(v_max_accum, zigzag); // Update max value seen so far } max_d = _mm512_reduce_max_epu32(v_max_accum); // Horizontal max reduction if (j > 0) prev = zxc_le32(in_ptr + (j - 1) * 4); } #elif defined(ZXC_USE_AVX2) if (frames >= 8) { __m256i v_max_accum = _mm256_setzero_si256(); // Initialize max accumulator to 0 for (; j < (frames & ~7); j += 8) { if (UNLIKELY(i == 0 && j == 0)) goto _scalar; // Load 8 consecutive integers const __m256i vc = _mm256_loadu_si256((const __m256i*)(in_ptr + j * 4)); // Load 8 integers offset by -1 const __m256i vp = _mm256_loadu_si256((const __m256i*)(in_ptr + j * 4 - 4)); const __m256i diff = _mm256_sub_epi32(vc, vp); // Compute deltas // ZigZag encode: (diff << 1) ^ (diff >> 31) const __m256i zigzag = _mm256_xor_si256(_mm256_slli_epi32(diff, 1), _mm256_srai_epi32(diff, 31)); _mm256_storeu_si256((__m256i*)&deltas[j], zigzag); // Store results v_max_accum = _mm256_max_epu32(v_max_accum, zigzag); // Update max accumulator } max_d = zxc_mm256_reduce_max_epu32(v_max_accum); // Horizontal max reduction if (j > 0) { prev = zxc_le32(in_ptr + (j - 1) * 4); } } #elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) // NEON processes 128-bit vectors (4 uint32 integers) if (frames >= 4) { uint32x4_t v_max_accum = vdupq_n_u32(0); // Initialize vector with zeros for (; j < (frames & ~3); j += 4) { if (UNLIKELY(i == 0 && j == 0)) goto _scalar; // Load 4 32-bit integers const uint32x4_t vc = vld1q_u32((const uint32_t*)(in_ptr + j * 4)); const uint32x4_t vp = vld1q_u32((const uint32_t*)(in_ptr + j * 4 - 4)); const uint32x4_t diff = vsubq_u32(vc, vp); // Calc deltas // ZigZag encode: (diff << 1) ^ (diff >> 31) const uint32x4_t z1 = vshlq_n_u32(diff, 1); // Arithmetic shift right to duplicate sign bit const uint32x4_t z2 = vreinterpretq_u32_s32(vshrq_n_s32(vreinterpretq_s32_u32(diff), 31)); const uint32x4_t zigzag = veorq_u32(z1, z2); vst1q_u32(&deltas[j], zigzag); // Store results v_max_accum = vmaxq_u32(v_max_accum, zigzag); // Update max accumulator } #if defined(ZXC_USE_NEON64) max_d = vmaxvq_u32(v_max_accum); // Reduce vector to single max value (AArch64) #else // NEON 32-bit (ARMv7) fallback for horizontal max using standard shifts // Reduce 4 elements -> 2 uint32x4_t v_swap = vextq_u32(v_max_accum, v_max_accum, 2); // Swap low/high 64-bit halves uint32x4_t v_max2 = vmaxq_u32(v_max_accum, v_swap); // Reduce 2 -> 1 v_swap = vextq_u32(v_max2, v_max2, 1); // Shift by 32 bits uint32x4_t v_max1 = vmaxq_u32(v_max2, v_swap); max_d = vgetq_lane_u32(v_max1, 0); #endif if (j > 0) prev = zxc_le32(in_ptr + (j - 1) * sizeof(uint32_t)); } #endif #if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) || defined(ZXC_USE_NEON64) || \ defined(ZXC_USE_NEON32) _scalar: #endif for (; j < frames; j++) { const uint32_t v = zxc_le32(in_ptr + j * sizeof(uint32_t)); const uint32_t diff = zxc_zigzag_encode((int32_t)(v - prev)); deltas[j] = diff; if (diff > max_d) max_d = diff; prev = v; } in_ptr += frames * sizeof(uint32_t); const uint8_t bits = zxc_highbit32(max_d); const size_t packed = ((frames * bits) + CHAR_BIT - 1) / CHAR_BIT; if (UNLIKELY(rem < ZXC_NUM_CHUNK_HEADER_SIZE + packed + sizeof(uint32_t))) return ZXC_ERROR_DST_TOO_SMALL; zxc_store_le16(p_curr, (uint16_t)frames); zxc_store_le16(p_curr + 2, bits); zxc_store_le64(p_curr + 4, (uint64_t)base); zxc_store_le32(p_curr + 12, (uint32_t)packed); p_curr += ZXC_NUM_CHUNK_HEADER_SIZE; rem -= ZXC_NUM_CHUNK_HEADER_SIZE; const int pb = zxc_bitpack_stream_32(deltas, frames, p_curr, rem, bits); if (UNLIKELY(pb < 0)) return pb; p_curr += pb; rem -= pb; } bh.comp_size = (uint32_t)(p_curr - (dst + ZXC_BLOCK_HEADER_SIZE)); const int hw = zxc_write_block_header(dst, dst_cap, &bh); if (UNLIKELY(hw < 0)) return hw; // Checksum will be appended by the wrapper *out_sz = ZXC_BLOCK_HEADER_SIZE + bh.comp_size; return ZXC_OK; } /** * @brief Update dp[p + L_start .. p + L_end) with a constant transition * cost, in parallel where the target ISA allows. * * For each L in [L_start, L_end), if @p nxt is strictly less than the * current dp[p+L], rewrite dp/parent_len/parent_off in lockstep: same * semantics as the scalar update inside ::zxc_lz77_optimal_parse_glo. * Caller guarantees @p nxt is independent of L (the cost of the L-th * transition does not vary across the requested span). * * Vectorized prologue per ISA, falling through to a scalar tail: * - AVX-512 BW + VL : 16-wide via vpcmpud + vmask{store,storeu}. * Falls back to AVX2 if VL is absent. * - AVX2 : 8-wide via biased vpcmpgt + vpblendvb (no 32-bit * unsigned cmpgt before AVX-512). parent_off is * updated with a packed 8x16 mask + 128-bit blend. * - NEON64 / NEON32 : 4-wide via vcgtq_u32 + vbslq_u32, with vmovn_u32 * to narrow the mask for the 4x16 parent_off update. * * @param[in,out] dp DP cost array; dp[p + L] is relaxed when * @p nxt < dp[p + L]. * @param[in,out] parent_len Backtrack length array, written in lockstep * with @p dp; receives the L of the relaxing * transition. * @param[in,out] parent_off Backtrack offset array, written in lockstep; * receives @p off_biased on relaxation. * @param[in] p Source DP position the transitions originate * from. Indexing into the three arrays is * `p + L`. * @param[in] L Initial L value (start of the span, inclusive). * @param[in] L_end End of the span (exclusive). Must satisfy * @p L_end <= UINT16_MAX so every written length * fits in @c parent_len's @c uint16_t cells. * @param[in] nxt Constant successor cost `dp[p] + transition`, * shared across the [L, L_end) span. * @param[in] off_biased Match offset minus ::ZXC_LZ_OFFSET_BIAS, the * value stored when a transition wins. * @return The first L value not processed (i.e., @p L_end on success). */ // codeql[cpp/unused-static-function]: false positive static ZXC_ALWAYS_INLINE size_t zxc_opt_dp_update_const_cost( uint32_t* RESTRICT dp, uint16_t* RESTRICT parent_len, uint16_t* RESTRICT parent_off, const size_t p, size_t L, const size_t L_end, const uint32_t nxt, const uint16_t off_biased) { #if defined(ZXC_USE_AVX512) && defined(__AVX512VL__) if (L + 16 <= L_end) { const __m512i v_inc = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); const __m512i v_nxt = _mm512_set1_epi32((int)nxt); const __m256i v_off = _mm256_set1_epi16((int16_t)off_biased); for (; L + 16 <= L_end; L += 16) { const __m512i v_L_lanes = _mm512_add_epi32(v_inc, _mm512_set1_epi32((int)L)); const __m512i v_dp = _mm512_loadu_si512((const void*)&dp[p + L]); const __mmask16 m = _mm512_cmplt_epu32_mask(v_nxt, v_dp); _mm512_mask_storeu_epi32(&dp[p + L], m, v_nxt); const __m256i v_L_u16 = _mm512_cvtusepi32_epi16(v_L_lanes); _mm256_mask_storeu_epi16((void*)&parent_len[p + L], m, v_L_u16); _mm256_mask_storeu_epi16((void*)&parent_off[p + L], m, v_off); } } #elif defined(ZXC_USE_AVX2) if (L + 8 <= L_end) { const __m256i v_inc = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7); const __m256i v_nxt = _mm256_set1_epi32((int)nxt); const __m256i v_bias = _mm256_set1_epi32((int)0x80000000); const __m256i v_nxt_b = _mm256_xor_si256(v_nxt, v_bias); const __m128i v_off = _mm_set1_epi16((int16_t)off_biased); for (; L + 8 <= L_end; L += 8) { const __m256i v_L_lanes = _mm256_add_epi32(v_inc, _mm256_set1_epi32((int)L)); const __m256i v_dp = _mm256_loadu_si256((const __m256i*)&dp[p + L]); /* Unsigned-compare-via-bias trick: * (dp ^ 0x80000000) > (nxt ^ 0x80000000) iff dp > nxt * because XOR with the sign bit maps unsigned ordering to * signed ordering. AVX2 only has signed cmpgt for 32-bit. */ const __m256i v_dp_b = _mm256_xor_si256(v_dp, v_bias); const __m256i v_mask = _mm256_cmpgt_epi32(v_dp_b, v_nxt_b); const __m256i v_dp_new = _mm256_blendv_epi8(v_dp, v_nxt, v_mask); _mm256_storeu_si256((__m256i*)&dp[p + L], v_dp_new); /* Pack 8x int32 mask -> 8x int16 mask with signed saturation: * 0xFFFFFFFF -> 0xFFFF, 0x00000000 -> 0x0000. */ const __m128i v_mask16 = _mm_packs_epi32(_mm256_castsi256_si128(v_mask), _mm256_extracti128_si256(v_mask, 1)); const __m128i v_L_u16 = _mm_packus_epi32(_mm256_castsi256_si128(v_L_lanes), _mm256_extracti128_si256(v_L_lanes, 1)); const __m128i v_pl = _mm_loadu_si128((const __m128i*)&parent_len[p + L]); const __m128i v_pl_new = _mm_blendv_epi8(v_pl, v_L_u16, v_mask16); _mm_storeu_si128((__m128i*)&parent_len[p + L], v_pl_new); const __m128i v_po = _mm_loadu_si128((const __m128i*)&parent_off[p + L]); const __m128i v_po_new = _mm_blendv_epi8(v_po, v_off, v_mask16); _mm_storeu_si128((__m128i*)&parent_off[p + L], v_po_new); } } #elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) if (L + 4 <= L_end) { static const uint32_t k_inc_array[4] = {0, 1, 2, 3}; const uint32x4_t v_inc = vld1q_u32(k_inc_array); const uint32x4_t v_nxt = vdupq_n_u32(nxt); const uint16x4_t v_off = vdup_n_u16(off_biased); for (; L + 4 <= L_end; L += 4) { const uint32x4_t v_L_lanes = vaddq_u32(v_inc, vdupq_n_u32((uint32_t)L)); const uint32x4_t v_dp = vld1q_u32(&dp[p + L]); const uint32x4_t v_mask = vcgtq_u32(v_dp, v_nxt); vst1q_u32(&dp[p + L], vbslq_u32(v_mask, v_nxt, v_dp)); const uint16x4_t v_mask16 = vmovn_u32(v_mask); const uint16x4_t v_L_u16 = vqmovn_u32(v_L_lanes); const uint16x4_t v_pl = vld1_u16(&parent_len[p + L]); vst1_u16(&parent_len[p + L], vbsl_u16(v_mask16, v_L_u16, v_pl)); const uint16x4_t v_po = vld1_u16(&parent_off[p + L]); vst1_u16(&parent_off[p + L], vbsl_u16(v_mask16, v_off, v_po)); } } #endif /* Scalar tail (and full path on archs without SIMD). * L < L_end <= UINT16_MAX (caller precondition), so the cast is lossless. */ for (; L < L_end; L++) { if (nxt < dp[p + L]) { dp[p + L] = nxt; parent_len[p + L] = (uint16_t)L; parent_off[p + L] = off_biased; } } return L; } /** * @brief Estimate per-block literal cost from a sampled histogram passed * through the actual length-limited Huffman builder. * * Strategy: build a strided sample of @p src (4096 entries), run the same * length-limited Huffman code construction the encoder uses, and report the * sample-weighted average code length. This is the predicted bits/byte * for Huffman-encoded literals on this distribution: no calibration * constants, no per-corpus tuning. The cap at 8 reflects that RAW is * always available at exactly that cost; if Huffman doesn't beat 8 on the * sample, the encoder will pick RAW and 8 is the right price. * * @param[in] src Source buffer for the block. * @param[in] src_sz Length of @p src in bytes. * @param[in] scratch Package-merge scratch (pre-allocated in the cctx for * level >= 6). May be `NULL`, in which case the builder * allocates its own working memory. * @return Estimated literal cost in bits, in `[1, 8]`. */ // codeql[cpp/unused-static-function]: false positive static uint32_t zxc_opt_estimate_lit_bits(const uint8_t* RESTRICT src, const size_t src_sz, void* RESTRICT scratch) { if (UNLIKELY(src_sz < ZXC_OPT_LIT_SAMPLE_MIN)) return CHAR_BIT; uint32_t hist[ZXC_HUF_NUM_SYMBOLS] = {0}; const size_t step = (src_sz > 4096) ? (src_sz >> 12) : 1U; size_t sampled = 0; for (size_t i = 0; i < src_sz; i += step) { hist[src[i]]++; sampled++; } uint8_t code_len[ZXC_HUF_NUM_SYMBOLS]; if (UNLIKELY(zxc_huf_build_code_lengths(hist, code_len, scratch) != ZXC_OK)) return CHAR_BIT; /* Sample-weighted sum of code lengths == predicted total Huffman bits * for the sample. Divide by sample count for bits/byte, rounded up * (DP works in integer bits; rounding up errs on the conservative * side, slightly favoring matches over fractional-cost literals). */ uint64_t total_bits = 0; for (int k = 0; k < ZXC_HUF_NUM_SYMBOLS; k++) { total_bits += (uint64_t)hist[k] * (uint64_t)code_len[k]; } const uint32_t avg = (uint32_t)((total_bits + sampled - 1) / sampled); /* Cap at RAW cost: if Huffman can't beat 8 bits/byte on the sample, * the encoder will pick RAW anyway and 8 is the actual literal cost. */ return (avg < CHAR_BIT) ? avg : CHAR_BIT; } /** * @brief Static price-based optimal LZ77 parser for level 6. * * Forward DP over the block's positions: `dp[p]` = min bit cost to encode * `src[0..p)`. Per-position transitions are * - literal: `dp[p+1] = min(dp[p+1], dp[p] + lit_cost` * - match : `dp[p+L] = min(dp[p+L], dp[p] + match_cost(L))` for L in * `[MIN_MATCH, max_L]` * where `max_L` is the longest match found by ::zxc_lz77_find_best_match at * `p` (with lazy disabled, the DP itself handles position-based * optimization). Backtracking from `dp[src_sz]` reconstructs the * optimal token sequence. * * Complexity guard: ::ZXC_OPT_LONG_MATCH_SKIP causes ::zxc_lz77_find_best_match * to be skipped at positions strictly inside a long match, without this * guard, highly repetitive data (e.g. Lorem-loop with multi-MB matches at * every offset) makes the parser quadratic and unit tests run for minutes. * The inner sub-length update loop visits every L from `MIN_MATCH` to * `max_L`; the skip threshold means each long-match region only pays its * O(L) cost once at the starting position, keeping total work O(N). * * @param[in,out] ctx Compression context. The lazy-allocated * `opt_scratch` field provides the DP arrays; * it is grown on first use and reused on * subsequent blocks. * @param[in] src Source buffer to parse. * @param[in] src_sz Length of @p src in bytes. * @param[in,out] hash_table LZ77 hash table (epoch | position entries). * @param[in,out] hash_tags 8-bit fast-rejection tags paired with @p hash_table. * @param[in,out] chain_table Hash-chain link table (ring buffer). * @param[in] epoch_mark Current epoch shifted into the high bits. * @param[in] offset_mask Mask isolating the position bits in chain entries. * @param[in] level Compression level (used to size the matcher). * @param[out] literals Buffer receiving the gathered literal bytes. * @param[out] buf_tokens Buffer receiving the per-sequence token bytes. * @param[out] buf_offsets Buffer receiving the per-sequence offsets. * @param[out] buf_extras Buffer receiving variable-length overflow data. * @param[out] seq_c_out Number of emitted sequences. * @param[out] lit_c_out Number of literal bytes written into @p literals. * @param[out] extras_sz_out Number of bytes written into @p buf_extras. * @param[out] max_offset_out Largest biased offset emitted (used by the caller * to choose 1-byte vs 2-byte offset encoding). * * @return `ZXC_OK` on success, or `ZXC_ERROR_MEMORY` if the DP scratch * allocations fail. */ static int zxc_lz77_optimal_parse_glo(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint32_t* RESTRICT hash_table, uint8_t* RESTRICT hash_tags, uint16_t* RESTRICT chain_table, const uint32_t epoch_mark, const uint32_t offset_mask, const int level, uint8_t* RESTRICT literals, uint8_t* RESTRICT buf_tokens, uint16_t* RESTRICT buf_offsets, uint8_t* RESTRICT buf_extras, uint32_t* RESTRICT seq_c_out, size_t* RESTRICT lit_c_out, size_t* RESTRICT extras_sz_out, uint16_t* RESTRICT max_offset_out) { zxc_lz77_params_t lzp_opt = zxc_get_lz77_params(level); lzp_opt.use_lazy = 0; // guard const uint8_t* const iend = src + src_sz; /* Block too small for any match: emit all as literals. */ if (UNLIKELY(src_sz < 13)) { if (src_sz > 0) ZXC_MEMCPY(literals, src, src_sz); *lit_c_out = src_sz; *seq_c_out = 0; *extras_sz_out = 0; *max_offset_out = 0; return ZXC_OK; } const size_t mflimit_pos = src_sz - 12; const uint8_t* const mflimit = src + mflimit_pos; /* DP arrays carved from ctx->opt_scratch: a single allocation lazy- * grown on the first level-6 call and reused across blocks. Each * sub-buffer is cache-line padded so the next one starts on a 64 B * boundary. The total `needed` matches zxc_estimate_cctx_size() keep * the formula in sync. * * dp : (chunk+1) x uint32_t: min cost to reach position p. * parent_len : (chunk+1) x uint16_t: 0 = literal, >= MIN_MATCH = match. * parent_off : (chunk+1) x uint16_t: biased match offset (distance-1). * match_end_bits : ceil((chunk+1)/64) x uint64_t: 1 bit per position, * set when that position * is the end of a match * on the chosen DP path. * Replaces a forward-order * actions[] stack at 1/64 * the cost. * * The same buffer is reused as transient scratch for the length-limited * Huffman code-length builder (see zxc_opt_estimate_lit_bits below and * the Huffman selection in zxc_encode_block_glo): the package-merge * scratch is needed before the DP runs and again after the parse has * been read out, so the lifetimes never overlap. The capacity is the * larger of the two demands. */ const size_t chunk = ctx->chunk_size; const size_t sz_dp = ZXC_ALIGN_CL((chunk + 1) * sizeof(uint32_t)); const size_t sz_pl = ZXC_ALIGN_CL((chunk + 1) * sizeof(uint16_t)); const size_t sz_po = ZXC_ALIGN_CL((chunk + 1) * sizeof(uint16_t)); const size_t n_bm_words = (chunk + 1 + 63) / 64; const size_t sz_bm = ZXC_ALIGN_CL(n_bm_words * sizeof(uint64_t)); const size_t dp_needed = sz_dp + sz_pl + sz_po + sz_bm; const size_t needed = (dp_needed > ZXC_HUF_BUILD_SCRATCH_SIZE) ? dp_needed : ZXC_HUF_BUILD_SCRATCH_SIZE; if (UNLIKELY(ctx->opt_scratch_cap < needed)) { if (ctx->opt_scratch) zxc_aligned_free(ctx->opt_scratch); ctx->opt_scratch = (uint8_t*)zxc_aligned_malloc(needed, ZXC_CACHE_LINE_SIZE); if (UNLIKELY(!ctx->opt_scratch)) { ctx->opt_scratch_cap = 0; return ZXC_ERROR_MEMORY; } ctx->opt_scratch_cap = needed; } /* Per-block literal cost: */ const uint32_t lit_cost = zxc_opt_estimate_lit_bits(src, src_sz, ctx->opt_scratch); uint32_t* const dp = (uint32_t*)ctx->opt_scratch; uint16_t* const parent_len = (uint16_t*)(ctx->opt_scratch + sz_dp); uint16_t* const parent_off = (uint16_t*)(ctx->opt_scratch + sz_dp + sz_pl); uint64_t* const match_end_bits = (uint64_t*)(ctx->opt_scratch + sz_dp + sz_pl + sz_po); dp[0] = 0; ZXC_MEMSET(dp + 1, 0xFF, src_sz * sizeof(uint32_t)); ZXC_MEMSET(parent_len, 0, sz_pl + sz_po + sz_bm); /* Forward DP: visit every position, update reachable successors. * `skip_until` skips find_best_match at positions strictly inside the * last long match, the DP transition from the start of the match * already covers dp[p+1..p+L], and re-searching at every intra-match * position is what makes the parser quadratic on repetitive inputs. */ size_t skip_until = 0; for (size_t p = 0; p < mflimit_pos; p++) { if (UNLIKELY(dp[p] == UINT32_MAX)) continue; /* Literal transition. */ const uint32_t lit_next = dp[p] + lit_cost; if (lit_next < dp[p + 1]) { dp[p + 1] = lit_next; parent_len[p + 1] = 0; } if (p < skip_until) continue; /* Match transition: call find_best_match (no lazy, no backtrack via * anchor=ip). Iterate sub-lengths since any L <= max_L matches at the * same offset and may end at a more useful DP position. */ const uint8_t* ip = src + p; const zxc_match_t m = zxc_lz77_find_best_match(src, ip, iend, mflimit, /*anchor=*/ip, hash_table, hash_tags, chain_table, epoch_mark, offset_mask, level, lzp_opt); if (m.ref) { const uint32_t off = (uint32_t)(ip - m.ref); if (off > 0 && off <= ZXC_LZ_WINDOW_SIZE) { const size_t L_max_raw = (m.len > src_sz - p) ? (src_sz - p) : (size_t)m.len; const size_t L_max = (L_max_raw > UINT16_MAX) ? UINT16_MAX : L_max_raw; /* The L-iteration cost function is piecewise constant in * varint segments. Split the [MIN_MATCH, L_max] span into: * 1. cheap : v < ML_MASK -> cost = base * 2. varint1 : v in [ML_MASK, ML_MASK + 128) -> cost = base + 8 * 3. varint2+: v >= ML_MASK + 128 -> cost = base + 16, +24, ... * * Steps 1 and 2 use constant nxt and are vectorized via * the helper. Step 3 is rare (typical matches are short) * and stays scalar. */ const uint16_t off_biased = (uint16_t)(off - ZXC_LZ_OFFSET_BIAS); const size_t L_max_plus = L_max + 1; size_t L = ZXC_LZ_MIN_MATCH_LEN; /* 1. Cheap range. */ { const size_t L_cheap_end = ZXC_LZ_MIN_MATCH_LEN + ZXC_TOKEN_ML_MASK; const size_t L_end = (L_max_plus < L_cheap_end) ? L_max_plus : L_cheap_end; const uint32_t nxt = dp[p] + ZXC_OPT_MATCH_COST_BASE; L = zxc_opt_dp_update_const_cost(dp, parent_len, parent_off, p, L, L_end, nxt, off_biased); } /* 2. First varint level (1-byte extension). */ if (L < L_max_plus) { const size_t L_v1_end = ZXC_LZ_MIN_MATCH_LEN + ZXC_TOKEN_ML_MASK + 128; const size_t L_end = (L_max_plus < L_v1_end) ? L_max_plus : L_v1_end; const uint32_t nxt = dp[p] + ZXC_OPT_MATCH_COST_BASE + CHAR_BIT; L = zxc_opt_dp_update_const_cost(dp, parent_len, parent_off, p, L, L_end, nxt, off_biased); } /* 3. Higher varint levels: variable cost, kept scalar. * Reached only by L >= ML_MASK + 128 + MIN_MATCH, so the * v >= ML_MASK guard from the original loop is implied. */ for (; L < L_max_plus; L++) { uint32_t cost = ZXC_OPT_MATCH_COST_BASE; uint32_t v = (uint32_t)(L - ZXC_LZ_MIN_MATCH_LEN) - ZXC_TOKEN_ML_MASK; cost += CHAR_BIT; while (v >= 128) { v >>= 7; cost += CHAR_BIT; } const uint32_t nxt = dp[p] + cost; if (nxt < dp[p + L]) { dp[p + L] = nxt; parent_len[p + L] = (uint16_t)L; parent_off[p + L] = off_biased; } } if (UNLIKELY(L_max >= ZXC_OPT_LONG_MATCH_SKIP)) skip_until = p + L_max - 1; } } } /* Last 12 bytes can only be literals (matches must end before iend). */ for (size_t p = mflimit_pos; p < src_sz; p++) { if (UNLIKELY(dp[p] == UINT32_MAX)) continue; const uint32_t lit_next = dp[p] + lit_cost; if (lit_next < dp[p + 1]) { dp[p + 1] = lit_next; parent_len[p + 1] = 0; } } /* Backtrack from src_sz to 0: only match endpoints are recorded (one bit * per position in match_end_bits). Literals between matches are implicit * runs of unmarked positions and are reconstructed during forward emission * via lit_start tracking, so they need no backtrack storage. */ { size_t pos = src_sz; while (pos > 0) { const uint32_t L = parent_len[pos]; if (L == 0) { pos -= 1; } else { match_end_bits[pos >> 6] |= (uint64_t)1 << (pos & 63); pos -= L; } } } /* Forward emission: walk match_end_bits word-by-word, peeling set bits * with ctzll. Each set bit gives a match endpoint; parent_len/parent_off * at that position recover (length, offset). */ uint32_t seq_c = 0; size_t lit_c = 0; size_t extras_sz = 0; uint16_t max_offset = 0; size_t lit_start = 0; for (size_t word_idx = 0; word_idx < n_bm_words; word_idx++) { uint64_t w = match_end_bits[word_idx]; while (w) { const size_t pos = (word_idx << 6) + (size_t)zxc_ctz64(w); w &= w - 1; const uint32_t L = parent_len[pos]; const uint16_t off_biased = parent_off[pos]; const size_t match_start = pos - L; const size_t LL = match_start - lit_start; if (LL > 0) { ZXC_MEMCPY(literals + lit_c, src + lit_start, LL); lit_c += LL; } const uint32_t ll = (uint32_t)LL; const uint32_t ml = L - ZXC_LZ_MIN_MATCH_LEN; const uint8_t ll_code = (ll >= ZXC_TOKEN_LL_MASK) ? ZXC_TOKEN_LL_MASK : (uint8_t)ll; const uint8_t ml_code = (ml >= ZXC_TOKEN_ML_MASK) ? ZXC_TOKEN_ML_MASK : (uint8_t)ml; buf_tokens[seq_c] = (ll_code << ZXC_TOKEN_LIT_BITS) | ml_code; buf_offsets[seq_c] = off_biased; if (off_biased > max_offset) max_offset = off_biased; if (UNLIKELY(ll >= ZXC_TOKEN_LL_MASK)) extras_sz += zxc_write_varint(buf_extras + extras_sz, ll - ZXC_TOKEN_LL_MASK); if (UNLIKELY(ml >= ZXC_TOKEN_ML_MASK)) extras_sz += zxc_write_varint(buf_extras + extras_sz, ml - ZXC_TOKEN_ML_MASK); seq_c++; lit_start = pos; } } /* Tail literals after the last match (or all literals if no match). */ if (lit_start < src_sz) { const size_t tail = src_sz - lit_start; ZXC_MEMCPY(literals + lit_c, src + lit_start, tail); lit_c += tail; } *seq_c_out = seq_c; *lit_c_out = lit_c; *extras_sz_out = extras_sz; *max_offset_out = max_offset; return ZXC_OK; } /** * @brief Encodes a data block using the General (GLO) compression format. * * This function implements the core LZ77 compression logic. It dynamically * adjusts compression parameters (search depth, lazy matching strategy, and * step skipping) based on the compression level configured in the context. * * **LZ77 Implementation Details:** * 1. **Hash Chain:** Uses a hash table (`ctx->hash_table`) to find potential * match positions. Collisions are handled via a `chain_table`, allowing us to * search deeper into the history for a better match. * 2. **Lazy Matching:** If a match is found, we check the *next* byte to see if * it produces a longer match. If so, we output a literal and take the better * match. This is enabled for levels >= 3. * 3. **Step Skipping:** For lower levels (1-3), we skip bytes when updating the * hash table to increase speed (`step > 1`). For levels 4+, we process every * byte to maximize compression ratio. * 4. **SIMD Match Finding:** Uses AVX2/AVX512/NEON to compare 32/64 bytes at a * time during match length calculation, significantly speeding up long match * verification. * 5. **RLE Detection:** Analyzes literals to see if Run-Length Encoding would * be beneficial (saving > 10% space). * * The encoding process consists of: * 1. **LZ77 Parsing**: The function iterates through the source data, * maintaining a hash chain to find repeated patterns (matches). It supports * "Lazy Matching" for higher compression levels to optimize match selection. * 2. **Sequence Storage**: Matches are converted into sequences consisting of * literal lengths, match lengths, and offsets. * 3. **Bitpacking & Serialization**: The sequences are analyzed to determine * optimal bit-widths. The function then writes the block header, encodes * literals (using Raw or RLE encoding), and bit-packs the sequence streams into * the destination buffer. * * @param[in,out] ctx Pointer to the compression context containing hash tables * and configuration. * @param[in] src Pointer to the input source data. * @param[in] src_sz Size of the input data in bytes. * @param[out] dst Pointer to the destination buffer where compressed data will * be written. * @param[in] dst_cap Maximum capacity of the destination buffer. * @param[out] out_sz [Out] Pointer to a variable that will receive the total size * of the compressed output. * * @return ZXC_OK on success, or a negative zxc_error_t code (e.g., ZXC_ERROR_DST_TOO_SMALL) if an * error occurs (e.g., buffer overflow). */ static int zxc_encode_block_glo(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, size_t dst_cap, size_t* RESTRICT out_sz) { const int level = ctx->compression_level; const zxc_lz77_params_t lzp = zxc_get_lz77_params(level); ctx->epoch++; if (UNLIKELY(ctx->epoch >= ctx->max_epoch)) { ZXC_MEMSET(ctx->hash_table, 0, ZXC_LZ_HASH_SIZE * sizeof(uint32_t)); ZXC_MEMSET(ctx->hash_tags, 0, ZXC_LZ_HASH_SIZE * sizeof(uint8_t)); ctx->epoch = 1; } const uint32_t offset_bits = ctx->offset_bits; const uint32_t offset_mask = ctx->offset_mask; const uint32_t epoch_mark = ctx->epoch << offset_bits; const uint8_t *ip = src, *iend = src + src_sz, *anchor = ip, *mflimit = iend - 12; uint32_t* const hash_table = ctx->hash_table; uint8_t* const hash_tags = ctx->hash_tags; uint16_t* const chain_table = ctx->chain_table; uint8_t* const literals = ctx->literals; uint8_t* const buf_tokens = ctx->buf_tokens; uint16_t* const buf_offsets = ctx->buf_offsets; uint8_t* const buf_extras = ctx->buf_extras; uint32_t seq_c = 0; size_t lit_c = 0; size_t extras_sz = 0; uint16_t max_offset = 0; // Track max offset for 1-byte/2-byte mode decision /* Level 6+: price-based optimal parser (fills outputs and skips the * lazy loop + last_lits handling below via `goto parse_done`). */ if (level >= ZXC_LEVEL_DENSITY) { const int rc_opt = zxc_lz77_optimal_parse_glo( ctx, src, src_sz, hash_table, hash_tags, chain_table, epoch_mark, offset_mask, level, literals, buf_tokens, buf_offsets, buf_extras, &seq_c, &lit_c, &extras_sz, &max_offset); if (UNLIKELY(rc_opt != ZXC_OK)) return rc_opt; goto parse_done; } while (LIKELY(ip < mflimit)) { const size_t dist = (size_t)(ip - anchor); size_t step = lzp.step_base + (dist >> lzp.step_shift); if (UNLIKELY(ip + step >= mflimit)) step = 1; if (LIKELY(ip + step + sizeof(uint64_t) <= iend)) { const uint64_t v_next = zxc_le64(ip + step); // cppcheck-suppress unreadVariable const uint32_t h_next = zxc_hash_func(v_next, 1); ZXC_PREFETCH_READ(&hash_tags[h_next]); ZXC_PREFETCH_READ(&hash_table[h_next]); } const zxc_match_t m = zxc_lz77_find_best_match(src, ip, iend, mflimit, anchor, hash_table, hash_tags, chain_table, epoch_mark, offset_mask, level, lzp); if (m.ref) { ip -= m.backtrack; const uint32_t ll = (uint32_t)(ip - anchor); const uint32_t ml = (uint32_t)(m.len - ZXC_LZ_MIN_MATCH_LEN); const uint32_t off = (uint32_t)(ip - m.ref); if (ll > 0) { if (LIKELY(anchor + ZXC_PAD_SIZE <= iend)) { zxc_copy32(literals + lit_c, anchor); if (UNLIKELY(ll > ZXC_PAD_SIZE)) { ZXC_MEMCPY(literals + lit_c + ZXC_PAD_SIZE, anchor + ZXC_PAD_SIZE, ll - ZXC_PAD_SIZE); } } else { ZXC_MEMCPY(literals + lit_c, anchor, ll); } lit_c += ll; } const uint8_t ll_code = (ll >= ZXC_TOKEN_LL_MASK) ? ZXC_TOKEN_LL_MASK : (uint8_t)ll; const uint8_t ml_code = (ml >= ZXC_TOKEN_ML_MASK) ? ZXC_TOKEN_ML_MASK : (uint8_t)ml; buf_tokens[seq_c] = (ll_code << ZXC_TOKEN_LIT_BITS) | ml_code; buf_offsets[seq_c] = (uint16_t)(off - ZXC_LZ_OFFSET_BIAS); if ((off - ZXC_LZ_OFFSET_BIAS) > max_offset) max_offset = (uint16_t)(off - ZXC_LZ_OFFSET_BIAS); if (ll >= ZXC_TOKEN_LL_MASK) extras_sz += zxc_write_varint(buf_extras + extras_sz, ll - ZXC_TOKEN_LL_MASK); if (ml >= ZXC_TOKEN_ML_MASK) extras_sz += zxc_write_varint(buf_extras + extras_sz, ml - ZXC_TOKEN_ML_MASK); seq_c++; if (m.len > 2 && level > ZXC_LEVEL_BALANCED) { const uint8_t* match_end = ip + m.len; if (match_end < iend - 7) { const uint32_t pos_u = (uint32_t)((match_end - 2) - src); const uint64_t val_u8 = zxc_le64(match_end - 2); const uint32_t val_u = (uint32_t)val_u8; const uint32_t h_u = zxc_hash_func(val_u8, 1); const uint32_t prev_head = hash_table[h_u]; const uint32_t prev_idx = (prev_head & ~offset_mask) == epoch_mark ? (prev_head & offset_mask) : 0; hash_table[h_u] = epoch_mark | pos_u; hash_tags[h_u] = (uint8_t)(val_u ^ (val_u >> 16)); chain_table[pos_u & ZXC_LZ_WINDOW_MASK] = (prev_idx > 0 && (pos_u - prev_idx) < ZXC_LZ_WINDOW_SIZE) ? (uint16_t)(pos_u - prev_idx) : 0; } } ip += m.len; anchor = ip; } else { ip += step; } } const size_t last_lits = iend - anchor; if (last_lits > 0) { ZXC_MEMCPY(literals + lit_c, anchor, last_lits); lit_c += last_lits; } parse_done:; // --- RLE ANALYSIS --- size_t rle_size = 0; int enc_lit = ZXC_SECTION_ENCODING_RAW; if (lit_c > 0) { const uint8_t* p = literals; const uint8_t* const p_end = literals + lit_c; const uint8_t* const p_end_4 = p_end - 3; // Safe limit for 4-byte lookahead while (LIKELY(p < p_end)) { const uint8_t b = *p; const uint8_t* run_start = p++; // Fast run counting with early SIMD exit #if defined(ZXC_USE_AVX512) const __m512i vb = _mm512_set1_epi8((char)b); while (p <= p_end - 64) { const __m512i v = _mm512_loadu_si512((const void*)p); const __mmask64 mask = _mm512_cmpeq_epi8_mask(v, vb); if (mask != 0xFFFFFFFFFFFFFFFFULL) { p += (size_t)zxc_ctz64(~mask); goto _run_done; } p += 64; } #elif defined(ZXC_USE_AVX2) const __m256i vb = _mm256_set1_epi8((char)b); while (p <= p_end - 32) { const __m256i v = _mm256_loadu_si256((const __m256i*)p); const uint32_t mask = (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, vb)); if (mask != 0xFFFFFFFF) { p += zxc_ctz32(~mask); goto _run_done; } p += 32; } #elif defined(ZXC_USE_NEON64) const uint8x16_t vb = vdupq_n_u8(b); while (p <= p_end - 16) { const uint8x16_t v = vld1q_u8(p); const uint8x16_t eq = vceqq_u8(v, vb); /* SHRN nibble-mask: see find_best_match above for rationale. */ const uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(eq), 4)), 0); if (LIKELY(mask == ~(uint64_t)0)) { p += 16; } else { p += (size_t)(zxc_ctz64(~mask) >> 2); goto _run_done; } } #elif defined(ZXC_USE_NEON32) uint8x16_t vb = vdupq_n_u8(b); while (p <= p_end - 16) { uint8x16_t v = vld1q_u8(p); uint8x16_t eq = vceqq_u8(v, vb); uint8x16_t not_eq = vmvnq_u8(eq); // 32-bit ARM NEON doesn't always support vgetq_lane_u64 / vreinterpretq_u64_u8 so // we treat the 128-bit vector as 4 x 32-bit lanes */ const uint32x4_t neq32 = vreinterpretq_u32_u8(not_eq); const uint32_t l0 = vgetq_lane_u32(neq32, 0); const uint32_t l1 = vgetq_lane_u32(neq32, 1); const uint64_t lo = ((uint64_t)l1 << 32) | l0; if (lo != 0) { p += (size_t)(zxc_ctz64(lo) >> 3); goto _run_done; } const uint32_t h0 = vgetq_lane_u32(neq32, 2); const uint32_t h1 = vgetq_lane_u32(neq32, 3); const uint64_t hi = ((uint64_t)h1 << 32) | h0; if (hi != 0) { p += 8 + (zxc_ctz64(hi) >> 3); goto _run_done; } p += 16; } #endif while (p < p_end && *p == b) p++; #if defined(ZXC_USE_AVX512) || defined(ZXC_USE_AVX2) || defined(ZXC_USE_NEON64) || \ defined(ZXC_USE_NEON32) _run_done:; #endif const size_t run = (size_t)(p - run_start); if (run >= 4) { // RLE run: 2 bytes per 131 values, then remainder // Branchless: full_chunks * 2 + remainder handling const size_t full_chunks = run / 131; const size_t rem = run - full_chunks * 131; // Avoid modulo rle_size += full_chunks * 2; // Remainder: if >= 4 -> 2 bytes (RLE), else 1 + rem (literal) if (rem >= 4) rle_size += 2; else if (rem > 0) rle_size += 1 + rem; } else { // Literal run: scan ahead with fast SIMD lookahead const uint8_t* lit_start = run_start; #if defined(ZXC_USE_AVX512) while (p <= p_end_4 - 64) { const __m512i v0 = _mm512_loadu_si512((const void*)p); const __m512i v1 = _mm512_loadu_si512((const void*)(p + 1)); const __m512i v2 = _mm512_loadu_si512((const void*)(p + 2)); const __m512i v3 = _mm512_loadu_si512((const void*)(p + 3)); const __mmask64 mask = _mm512_cmpeq_epi8_mask(v0, v1) & _mm512_cmpeq_epi8_mask(v1, v2) & _mm512_cmpeq_epi8_mask(v2, v3); if (mask != 0) { p += (size_t)zxc_ctz64(mask); goto _lit_done; } p += 64; } #elif defined(ZXC_USE_AVX2) while (p <= p_end_4 - 32) { __m256i v0 = _mm256_loadu_si256((const __m256i*)p); __m256i v1 = _mm256_loadu_si256((const __m256i*)(p + 1)); __m256i v2 = _mm256_loadu_si256((const __m256i*)(p + 2)); __m256i v3 = _mm256_loadu_si256((const __m256i*)(p + 3)); __m256i vend = _mm256_and_si256( _mm256_cmpeq_epi8(v0, v1), _mm256_and_si256(_mm256_cmpeq_epi8(v1, v2), _mm256_cmpeq_epi8(v2, v3))); uint32_t mask = (uint32_t)_mm256_movemask_epi8(vend); if (mask != 0) { p += zxc_ctz32(mask); goto _lit_done; } p += 32; } #elif defined(ZXC_USE_NEON64) while (p <= p_end_4 - 16) { uint8x16_t v0 = vld1q_u8(p); uint8x16_t v1 = vld1q_u8(p + 1); uint8x16_t v2 = vld1q_u8(p + 2); uint8x16_t v3 = vld1q_u8(p + 3); uint8x16_t eq = vandq_u8(vceqq_u8(v0, v1), vandq_u8(vceqq_u8(v1, v2), vceqq_u8(v2, v3))); /* Dual of the run scan: searching for the FIRST set * nibble (a position where 4 consecutive bytes match). * mask == 0 means no break found in this 16-byte * window. Same SHRN compression as elsewhere. */ const uint64_t mask = vget_lane_u64( vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(eq), 4)), 0); if (LIKELY(mask == 0)) { p += 16; } else { p += (size_t)(zxc_ctz64(mask) >> 2); goto _lit_done; } } #elif defined(ZXC_USE_NEON32) while (p <= p_end_4 - 16) { uint8x16_t v0 = vld1q_u8(p); uint8x16_t v1 = vld1q_u8(p + 1); uint8x16_t v2 = vld1q_u8(p + 2); uint8x16_t v3 = vld1q_u8(p + 3); uint8x16_t eq = vandq_u8(vceqq_u8(v0, v1), vandq_u8(vceqq_u8(v1, v2), vceqq_u8(v2, v3))); uint32x4_t eq32 = vreinterpretq_u32_u8(eq); uint32_t l0 = vgetq_lane_u32(eq32, 0); uint32_t l1 = vgetq_lane_u32(eq32, 1); uint64_t lo = ((uint64_t)l1 << 32) | l0; if (lo != 0) { p += (zxc_ctz64(lo) >> 3); goto _lit_done; } uint32_t h0 = vgetq_lane_u32(eq32, 2); uint32_t h1 = vgetq_lane_u32(eq32, 3); uint64_t hi = ((uint64_t)h1 << 32) | h0; if (hi != 0) { p += 8 + (zxc_ctz64(hi) >> 3); goto _lit_done; } p += 16; } #endif while (p < p_end_4) { // Check for RLE opportunity (4 identical bytes) if (UNLIKELY(p[0] == p[1] && p[1] == p[2] && p[2] == p[3])) break; p++; } // Handle remaining bytes near end while (p < p_end) { if (UNLIKELY(p + 3 < p_end && p[0] == p[1] && p[1] == p[2] && p[2] == p[3])) break; p++; } #if defined(ZXC_USE_AVX512) || defined(ZXC_USE_AVX2) || defined(ZXC_USE_NEON64) || \ defined(ZXC_USE_NEON32) _lit_done:; #endif const size_t lit_run = (size_t)(p - lit_start); // 1 header per 128 bytes + all data bytes // lit_run + ceil(lit_run / 128) rle_size += lit_run + ((lit_run + 127) >> 7); } } // Threshold: ~3% savings using integer math (97% ~= 1 - 1/32) if (rle_size < lit_c - (lit_c >> 5)) enc_lit = ZXC_SECTION_ENCODING_RLE; } /* Level >= 6: also evaluate Huffman as a 3rd literal-encoding candidate. * Build a histogram and length-limited canonical code lengths, compute the * exact byte size of the 4-way interleaved bitstream + 134-byte header, * and switch to HUFFMAN if it beats the current choice by >= 3%. */ uint8_t huf_code_len[ZXC_HUF_NUM_SYMBOLS]; size_t huf_total_size = SIZE_MAX; if (level >= ZXC_LEVEL_DENSITY && lit_c >= ZXC_HUF_MIN_LITERALS) { uint32_t freq0[ZXC_HUF_NUM_SYMBOLS] = {0}; uint32_t freq1[ZXC_HUF_NUM_SYMBOLS] = {0}; uint32_t freq2[ZXC_HUF_NUM_SYMBOLS] = {0}; uint32_t freq3[ZXC_HUF_NUM_SYMBOLS] = {0}; { size_t i = 0; for (; i + 4 <= lit_c; i += 4) { freq0[literals[i + 0]]++; freq1[literals[i + 1]]++; freq2[literals[i + 2]]++; freq3[literals[i + 3]]++; } for (; i < lit_c; i++) freq0[literals[i]]++; } uint32_t freq[ZXC_HUF_NUM_SYMBOLS]; for (int k = 0; k < ZXC_HUF_NUM_SYMBOLS; k++) { freq[k] = freq0[k] + freq1[k] + freq2[k] + freq3[k]; } if (zxc_huf_build_code_lengths(freq, huf_code_len, ctx->opt_scratch) == ZXC_OK) { const size_t Q = (lit_c + ZXC_HUF_NUM_STREAMS - 1) / ZXC_HUF_NUM_STREAMS; size_t streams_bytes = 0; for (int s = 0; s < ZXC_HUF_NUM_STREAMS; s++) { size_t start = (size_t)s * Q; size_t stop = start + Q; if (start > lit_c) start = lit_c; if (stop > lit_c) stop = lit_c; uint64_t b0 = 0, b1 = 0, b2 = 0, b3 = 0; size_t i = start; for (; i + 4 <= stop; i += 4) { b0 += huf_code_len[literals[i + 0]]; b1 += huf_code_len[literals[i + 1]]; b2 += huf_code_len[literals[i + 2]]; b3 += huf_code_len[literals[i + 3]]; } uint64_t bits = b0 + b1 + b2 + b3; for (; i < stop; i++) bits += huf_code_len[literals[i]]; streams_bytes += (size_t)((bits + 7) / 8); } huf_total_size = ZXC_HUF_HEADER_SIZE + streams_bytes; const size_t baseline = (enc_lit == ZXC_SECTION_ENCODING_RLE) ? rle_size : (size_t)lit_c; /* Threshold: 3% savings (1/32) over the chosen RAW/RLE baseline. * Same heuristic as the RAW/RLE switch above. */ if (huf_total_size < baseline - (baseline >> 5)) { enc_lit = ZXC_SECTION_ENCODING_HUFFMAN; } } } zxc_block_header_t bh = {.block_type = ZXC_BLOCK_GLO}; uint8_t* const p = dst + ZXC_BLOCK_HEADER_SIZE; size_t rem = dst_cap - ZXC_BLOCK_HEADER_SIZE; // Decide offset encoding mode: 1-byte if all offsets <= 255 const int use_8bit_off = (max_offset <= 255) ? 1 : 0; const size_t off_stream_size = use_8bit_off ? seq_c : (seq_c * 2); const zxc_gnr_header_t gh = {.n_sequences = seq_c, .n_literals = (uint32_t)lit_c, .enc_lit = enc_lit, .enc_litlen = 0, .enc_mlen = 0, .enc_off = (uint8_t)use_8bit_off}; zxc_section_desc_t desc[ZXC_GLO_SECTIONS] = {0}; const size_t lit_section_size = (enc_lit == ZXC_SECTION_ENCODING_RLE) ? rle_size : (enc_lit == ZXC_SECTION_ENCODING_HUFFMAN) ? huf_total_size : (size_t)lit_c; desc[0].sizes = (uint64_t)lit_section_size | ((uint64_t)lit_c << 32); desc[1].sizes = (uint64_t)seq_c | ((uint64_t)seq_c << 32); desc[2].sizes = (uint64_t)off_stream_size | ((uint64_t)off_stream_size << 32); desc[3].sizes = (uint64_t)extras_sz | ((uint64_t)extras_sz << 32); const int ghs = zxc_write_glo_header_and_desc(p, rem, &gh, desc); if (UNLIKELY(ghs < 0)) return ghs; uint8_t* p_curr = p + ghs; rem -= ghs; // Extract stream sizes once const size_t sz_lit = (size_t)(desc[0].sizes & ZXC_SECTION_SIZE_MASK); const size_t sz_tok = (size_t)(desc[1].sizes & ZXC_SECTION_SIZE_MASK); const size_t sz_off = (size_t)(desc[2].sizes & ZXC_SECTION_SIZE_MASK); const size_t sz_ext = (size_t)(desc[3].sizes & ZXC_SECTION_SIZE_MASK); if (UNLIKELY(rem < sz_lit)) return ZXC_ERROR_DST_TOO_SMALL; if (enc_lit == ZXC_SECTION_ENCODING_HUFFMAN) { const int written = zxc_huf_encode_section(literals, (size_t)lit_c, huf_code_len, p_curr, rem); if (UNLIKELY(written < 0)) return written; if (UNLIKELY((size_t)written != huf_total_size)) return ZXC_ERROR_DST_TOO_SMALL; p_curr += written; } else if (enc_lit == ZXC_SECTION_ENCODING_RLE) { // Write RLE - optimized single-pass encoding const uint8_t* lit_ptr = literals; const uint8_t* const lit_end = literals + lit_c; while (lit_ptr < lit_end) { uint8_t b = *lit_ptr; const uint8_t* run_start = lit_ptr++; // Count run length while (lit_ptr < lit_end && *lit_ptr == b) lit_ptr++; size_t run = (size_t)(lit_ptr - run_start); if (run >= 4) { // RLE runs: emit 2-byte tokens (header + value) while (run >= 4) { size_t chunk = (run > 131) ? 131 : run; *p_curr++ = (uint8_t)(ZXC_LIT_RLE_FLAG | (chunk - 4)); *p_curr++ = b; run -= chunk; } // Leftover < 4 bytes: emit as literal if (run > 0) { *p_curr++ = (uint8_t)(run - 1); ZXC_MEMCPY(p_curr, lit_ptr - run, run); p_curr += run; } } else { // Literal run: scan ahead to find next RLE opportunity const uint8_t* lit_run_start = run_start; while (lit_ptr < lit_end) { // Quick check: need 4 identical bytes to break if (UNLIKELY(lit_ptr + 3 < lit_end && lit_ptr[0] == lit_ptr[1] && lit_ptr[1] == lit_ptr[2] && lit_ptr[2] == lit_ptr[3])) { break; } lit_ptr++; } size_t lit_run = (size_t)(lit_ptr - lit_run_start); const uint8_t* src_ptr = lit_run_start; // Emit literal chunks (max 128 bytes each) while (lit_run > 0) { size_t chunk = (lit_run > 128) ? 128 : lit_run; *p_curr++ = (uint8_t)(chunk - 1); ZXC_MEMCPY(p_curr, src_ptr, chunk); p_curr += chunk; src_ptr += chunk; lit_run -= chunk; } } } } else { ZXC_MEMCPY(p_curr, literals, lit_c); p_curr += lit_c; } rem -= sz_lit; if (UNLIKELY(rem < sz_tok)) return ZXC_ERROR_DST_TOO_SMALL; ZXC_MEMCPY(p_curr, buf_tokens, seq_c); p_curr += seq_c; rem -= sz_tok; if (UNLIKELY(rem < sz_off)) return ZXC_ERROR_DST_TOO_SMALL; if (use_8bit_off) { // Write 1-byte offsets - unroll for better throughput uint32_t i = 0; for (; i + 8 <= seq_c; i += 8) { p_curr[0] = (uint8_t)buf_offsets[i + 0]; p_curr[1] = (uint8_t)buf_offsets[i + 1]; p_curr[2] = (uint8_t)buf_offsets[i + 2]; p_curr[3] = (uint8_t)buf_offsets[i + 3]; p_curr[4] = (uint8_t)buf_offsets[i + 4]; p_curr[5] = (uint8_t)buf_offsets[i + 5]; p_curr[6] = (uint8_t)buf_offsets[i + 6]; p_curr[7] = (uint8_t)buf_offsets[i + 7]; p_curr += 8; } for (; i < seq_c; i++) { *p_curr++ = (uint8_t)buf_offsets[i]; } } else { // Write 2-byte offsets in little-endian order #ifdef ZXC_BIG_ENDIAN for (uint32_t i = 0; i < seq_c; i++) { zxc_store_le16(p_curr, buf_offsets[i]); p_curr += sizeof(uint16_t); } #else ZXC_MEMCPY(p_curr, buf_offsets, seq_c * sizeof(uint16_t)); p_curr += seq_c * sizeof(uint16_t); #endif } rem -= sz_off; if (UNLIKELY(rem < sz_ext)) return ZXC_ERROR_DST_TOO_SMALL; ZXC_MEMCPY(p_curr, buf_extras, extras_sz); p_curr += extras_sz; bh.comp_size = (uint32_t)(p_curr - (dst + ZXC_BLOCK_HEADER_SIZE)); const int hw = zxc_write_block_header(dst, dst_cap, &bh); if (UNLIKELY(hw < 0)) return hw; // Checksum will be appended by the wrapper *out_sz = ZXC_BLOCK_HEADER_SIZE + bh.comp_size; return ZXC_OK; } /** * @brief Encodes a data block using the General High Velocity (GHI) compression format. * * 1. Compression Strategy * It uses an LZ77-based algorithm with a sliding window (64KB) and a hash table/chain table * mechanism. * * 2. Token Format (Fixed-Width) * Unlike the standard GLO block which uses 1-byte tokens (4-bit literal length / 4-bit match * length), GHI uses 4-byte (32-bit) sequence records for better performance on long runs: * Literal Length (LL): 8 bits (stores 0-254; 255 indicates overflow). * Match Length (ML): 8 bits (stores 0-254; 255 indicates overflow). * Offset: 16 bits (supports the full 64KB window). * This format minimizes the number of expensive VByte reads during decompression for common * sequences where lengths are between 16 and 255. * * @param[in,out] ctx Pointer to the compression context containing hash tables * and configuration. * @param[in] src Pointer to the input source data. * @param[in] src_sz Size of the input data in bytes. * @param[out] dst Pointer to the destination buffer where compressed data will * be written. * @param[in] dst_cap Maximum capacity of the destination buffer. * @param[out] out_sz Pointer to a variable that will receive the total size * of the compressed output. * * @return ZXC_OK on success, or a negative zxc_error_t code (e.g., ZXC_ERROR_DST_TOO_SMALL) if an * error occurs (e.g., buffer overflow). */ static int zxc_encode_block_ghi(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap, size_t* RESTRICT const out_sz) { const int level = ctx->compression_level; const zxc_lz77_params_t lzp = zxc_get_lz77_params(level); ctx->epoch++; if (UNLIKELY(ctx->epoch >= ctx->max_epoch)) { ZXC_MEMSET(ctx->hash_table, 0, ZXC_LZ_HASH_SIZE * sizeof(uint32_t)); ZXC_MEMSET(ctx->hash_tags, 0, ZXC_LZ_HASH_SIZE * sizeof(uint8_t)); ctx->epoch = 1; } const uint32_t offset_bits = ctx->offset_bits; const uint32_t offset_mask = ctx->offset_mask; const uint32_t epoch_mark = ctx->epoch << offset_bits; const uint8_t *ip = src, *iend = src + src_sz, *anchor = ip, *mflimit = iend - 12; uint32_t* const hash_table = ctx->hash_table; uint8_t* const hash_tags = ctx->hash_tags; uint8_t* const buf_extras = ctx->buf_extras; uint16_t* const chain_table = ctx->chain_table; uint8_t* const literals = ctx->literals; uint32_t* const buf_sequences = ctx->buf_sequences; uint32_t seq_c = 0; size_t extras_c = 0; size_t lit_c = 0; uint16_t max_offset = 0; while (LIKELY(ip < mflimit)) { size_t dist = (size_t)(ip - anchor); size_t step = lzp.step_base + (dist >> lzp.step_shift); if (UNLIKELY(ip + step >= mflimit)) step = 1; ZXC_PREFETCH_READ(ip + step * 4 + ZXC_CACHE_LINE_SIZE); if (LIKELY(ip + step + sizeof(uint64_t) <= iend)) { const uint64_t v_next = zxc_le64(ip + step); // cppcheck-suppress unreadVariable const uint32_t h_next = zxc_hash_func(v_next, 0); ZXC_PREFETCH_READ(&hash_tags[h_next]); ZXC_PREFETCH_READ(&hash_table[h_next]); } const zxc_match_t m = zxc_lz77_find_best_match(src, ip, iend, mflimit, anchor, hash_table, hash_tags, chain_table, epoch_mark, offset_mask, level, lzp); if (m.ref) { ip -= m.backtrack; const uint32_t ll = (uint32_t)(ip - anchor); const uint32_t ml = (uint32_t)(m.len - ZXC_LZ_MIN_MATCH_LEN); const uint32_t off = (uint32_t)(ip - m.ref); if (ll > 0) { if (LIKELY(anchor + ZXC_PAD_SIZE <= iend)) { zxc_copy32(literals + lit_c, anchor); if (UNLIKELY(ll > ZXC_PAD_SIZE)) { ZXC_MEMCPY(literals + lit_c + ZXC_PAD_SIZE, anchor + ZXC_PAD_SIZE, ll - ZXC_PAD_SIZE); } } else { ZXC_MEMCPY(literals + lit_c, anchor, ll); } lit_c += ll; } const uint32_t ll_write = (ll >= ZXC_SEQ_LL_MASK) ? 255U : ll; const uint32_t ml_write = (ml >= ZXC_SEQ_ML_MASK) ? 255U : ml; const uint32_t seq_val = (ll_write << (ZXC_SEQ_ML_BITS + ZXC_SEQ_OFF_BITS)) | (ml_write << ZXC_SEQ_OFF_BITS) | ((off - ZXC_LZ_OFFSET_BIAS) & ZXC_SEQ_OFF_MASK); if ((off - ZXC_LZ_OFFSET_BIAS) > max_offset) max_offset = (uint16_t)(off - ZXC_LZ_OFFSET_BIAS); buf_sequences[seq_c] = seq_val; seq_c++; if (ll >= ZXC_SEQ_LL_MASK) extras_c += zxc_write_varint(buf_extras + extras_c, ll - ZXC_SEQ_LL_MASK); if (ml >= ZXC_SEQ_ML_MASK) extras_c += zxc_write_varint(buf_extras + extras_c, ml - ZXC_SEQ_ML_MASK); ip += m.len; anchor = ip; } else { ip += step; } } const size_t last_lits = iend - anchor; if (last_lits > 0) { ZXC_MEMCPY(literals + lit_c, anchor, last_lits); lit_c += last_lits; } zxc_block_header_t bh = {.block_type = ZXC_BLOCK_GHI}; uint8_t* const p = dst + ZXC_BLOCK_HEADER_SIZE; size_t rem = dst_cap - ZXC_BLOCK_HEADER_SIZE; // Decide offset encoding mode const zxc_gnr_header_t gh = {.n_sequences = seq_c, .n_literals = (uint32_t)lit_c, .enc_lit = ZXC_SECTION_ENCODING_RAW, .enc_litlen = 0, .enc_mlen = 0, .enc_off = (uint8_t)(max_offset <= 255) ? 1 : 0}; zxc_section_desc_t desc[ZXC_GHI_SECTIONS] = {0}; desc[0].sizes = (uint64_t)lit_c | ((uint64_t)lit_c << 32); size_t sz_seqs = seq_c * sizeof(uint32_t); desc[1].sizes = (uint64_t)sz_seqs | ((uint64_t)sz_seqs << 32); desc[2].sizes = (uint64_t)extras_c | ((uint64_t)extras_c << 32); const int ghs = zxc_write_ghi_header_and_desc(p, rem, &gh, desc); if (UNLIKELY(ghs < 0)) return ghs; uint8_t* p_curr = p + ghs; rem -= ghs; // Extract stream sizes once const size_t sz_lit = (size_t)(desc[0].sizes & ZXC_SECTION_SIZE_MASK); const size_t sz_seq = (size_t)(desc[1].sizes & ZXC_SECTION_SIZE_MASK); const size_t sz_ext = (size_t)(desc[2].sizes & ZXC_SECTION_SIZE_MASK); if (UNLIKELY(rem < sz_lit + sz_seq + sz_ext)) return ZXC_ERROR_DST_TOO_SMALL; ZXC_MEMCPY(p_curr, literals, lit_c); p_curr += lit_c; rem -= sz_lit; if (UNLIKELY(rem < sz_seq)) return ZXC_ERROR_DST_TOO_SMALL; // Write sequences in little-endian order #ifdef ZXC_BIG_ENDIAN for (uint32_t i = 0; i < seq_c; i++) { zxc_store_le32(p_curr, buf_sequences[i]); p_curr += sizeof(uint32_t); } #else ZXC_MEMCPY(p_curr, buf_sequences, sz_seq); p_curr += sz_seq; #endif // --- WRITE EXTRAS --- ZXC_MEMCPY(p_curr, buf_extras, sz_ext); p_curr += sz_ext; bh.comp_size = (uint32_t)(p_curr - (dst + ZXC_BLOCK_HEADER_SIZE)); const int hw = zxc_write_block_header(dst, dst_cap, &bh); if (UNLIKELY(hw < 0)) return hw; // Checksum will be appended by the wrapper *out_sz = ZXC_BLOCK_HEADER_SIZE + bh.comp_size; return ZXC_OK; } /** * @brief Encodes a raw data block (uncompressed). * * This function prepares and writes a "RAW" type block into the destination * buffer. It handles the block header, copying of source data, and optionally * the calculation and storage of a checksum. * * @param[in] src Pointer to the source data to encode. * @param[in] src_sz Size of the source data in bytes. * @param[out] dst Pointer to the destination buffer. * @param[in] dst_cap Maximum capacity of the destination buffer. * @param[out] out_sz Pointer to a variable receiving the total written size * (header * + data + checksum). * @param[in] chk Boolean flag: if non-zero, a checksum is calculated and added. * * @return ZXC_OK on success, or a negative zxc_error_t code (e.g., ZXC_ERROR_DST_TOO_SMALL) if the * destination buffer capacity is insufficient. */ static int zxc_encode_block_raw(const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT const dst, const size_t dst_cap, size_t* RESTRICT const out_sz) { if (UNLIKELY(dst_cap < ZXC_BLOCK_HEADER_SIZE + src_sz)) return ZXC_ERROR_DST_TOO_SMALL; // Compute block RAW zxc_block_header_t bh; bh.block_type = ZXC_BLOCK_RAW; bh.block_flags = 0; // Checksum flag moved to file header bh.reserved = 0; bh.comp_size = (uint32_t)src_sz; const int hw = zxc_write_block_header(dst, dst_cap, &bh); if (UNLIKELY(hw < 0)) return hw; ZXC_MEMCPY(dst + ZXC_BLOCK_HEADER_SIZE, src, src_sz); // Checksum will be appended by the wrapper *out_sz = ZXC_BLOCK_HEADER_SIZE + src_sz; return ZXC_OK; } /** * @brief Checks if the given byte array represents a numeric value. * * This function examines the provided buffer to determine if it contains * only numeric characters (e.g., ASCII digits '0'-'9'). * * Improved heuristic: * 1. Must be aligned to 4 bytes. * 2. Samples the first 128 integers (more accurate). * 3. Calculates bit width of deltas (fewer bits = better for NUM). * 4. Estimates compression ratio: if NUM would save >20% vs raw, use it. * * @param[in] src Pointer to the input byte array to be checked. * @param[in] size The number of bytes in the input array. * @return int Returns 1 if the array is numeric, 0 otherwise. */ static int zxc_probe_is_numeric(const uint8_t* src, const size_t size) { if (UNLIKELY(size % sizeof(uint32_t) != 0 || size < (4 * sizeof(uint32_t)))) return 0; const size_t total_vals = size / sizeof(uint32_t); const size_t sample_len = 16; // Sample 2 contiguous regions: start and middle of the block. // Each region computes its own deltas independently. const size_t offsets[2] = {0, (total_vals / 2) & ~(size_t)3}; // Align to uint32_t boundary const size_t n_regions = (total_vals > sample_len * 2) ? 2 : 1; uint32_t max_zigzag = 0; uint32_t small_count = 0; // Deltas < 256 (8 bits) uint32_t medium_count = 0; // Deltas < 65536 (16 bits) size_t total_sampled = 0; for (size_t r = 0; r < n_regions; r++) { const uint8_t* p = src + offsets[r] * sizeof(uint32_t); const size_t region_count = ((total_vals - offsets[r]) < sample_len) ? (total_vals - offsets[r]) : sample_len; uint32_t prev = zxc_le32(p); p += sizeof(uint32_t); for (size_t i = 1; i < region_count; i++) { const uint32_t curr = zxc_le32(p); const int32_t diff = (int32_t)(curr - prev); const uint32_t zigzag = zxc_zigzag_encode(diff); max_zigzag = zigzag > max_zigzag ? zigzag : max_zigzag; small_count += (uint32_t)(zigzag < 256); medium_count += (uint32_t)(zigzag >= 256) & (uint32_t)(zigzag < 65536); prev = curr; p += sizeof(uint32_t); } total_sampled += region_count - 1; } const uint32_t bits_needed = zxc_highbit32(max_zigzag); // Estimate compression ratio: // NUM uses ~bits_needed per value, Raw uses 32 bits per value // Worth it if bits_needed <= 20 (saves >37.5%) if (bits_needed <= 16) return 1; if (bits_needed <= 20 && (small_count + medium_count) >= (total_sampled * 85) / 100) return 1; // Fallback: if 90% of deltas are small, still use NUM if ((small_count + medium_count) >= (total_sampled * 90) / 100) return 1; return 0; } // cppcheck-suppress unusedFunction int zxc_compress_chunk_wrapper(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT chunk, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { size_t w = 0; int res = ZXC_OK; int try_num = zxc_probe_is_numeric(chunk, src_sz); if (UNLIKELY(try_num)) { res = zxc_encode_block_num(ctx, chunk, src_sz, dst, dst_cap, &w); if (res != ZXC_OK || w > (src_sz - (src_sz >> 2))) // w > 75% of src_sz try_num = 0; // NUM didn't compress well, try GLO/GHI instead } if (LIKELY(!try_num)) { if (ctx->compression_level <= 2) res = zxc_encode_block_ghi(ctx, chunk, src_sz, dst, dst_cap, &w); else res = zxc_encode_block_glo(ctx, chunk, src_sz, dst, dst_cap, &w); } // Check expansion. W contains Header + Payload. if (UNLIKELY(res != ZXC_OK || w >= src_sz)) { res = zxc_encode_block_raw(chunk, src_sz, dst, dst_cap, &w); if (UNLIKELY(res != ZXC_OK)) return res; } if (ctx->checksum_enabled) { // Calculate checksum on the compressed payload (w currently excludes checksum) // Header is at dst, data starts at dst + ZXC_BLOCK_HEADER_SIZE if (UNLIKELY(w < ZXC_BLOCK_HEADER_SIZE || w + ZXC_BLOCK_CHECKSUM_SIZE > dst_cap)) return ZXC_ERROR_OVERFLOW; uint32_t payload_sz = (uint32_t)(w - ZXC_BLOCK_HEADER_SIZE); uint32_t crc = zxc_checksum(dst + ZXC_BLOCK_HEADER_SIZE, payload_sz, ZXC_CHECKSUM_RAPIDHASH); zxc_store_le32(dst + w, crc); w += ZXC_BLOCK_CHECKSUM_SIZE; } return (int)w; } zxc-0.11.0/src/lib/zxc_decompress.c000066400000000000000000002576071520102567100171460ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /** * @file zxc_decompress.c * @brief Block-level decompression: NUM / GLO / GHI / RAW decoding with * SIMD-accelerated prefix-sum, bit-unpacking, and overlapping copies. * * Like @ref zxc_compress.c, this file is compiled multiple times with * @c ZXC_FUNCTION_SUFFIX to produce per-ISA variants. */ /* * Function Multi-Versioning Support * If ZXC_FUNCTION_SUFFIX is defined (e.g. _avx2, _neon), rename the public * entry point AND the Huffman decoder consumed by this TU. The defines sit * before zxc_internal.h so that the prototypes the header declares are also * rewritten with the suffix, keeping callers and callees consistent. */ #ifdef ZXC_FUNCTION_SUFFIX #define ZXC_CAT_IMPL(x, y) x##y #define ZXC_CAT(x, y) ZXC_CAT_IMPL(x, y) #define zxc_decompress_chunk_wrapper ZXC_CAT(zxc_decompress_chunk_wrapper, ZXC_FUNCTION_SUFFIX) #define zxc_decompress_chunk_wrapper_safe \ ZXC_CAT(zxc_decompress_chunk_wrapper_safe, ZXC_FUNCTION_SUFFIX) #define zxc_huf_decode_section ZXC_CAT(zxc_huf_decode_section, ZXC_FUNCTION_SUFFIX) #endif #include "../../include/zxc_error.h" #include "../../include/zxc_sans_io.h" #include "zxc_internal.h" /** * @brief Consumes a specified number of bits from the bit reader buffer without * performing safety checks. * * This function advances the bit reader's state by `n` bits. It is marked as * always inline for performance critical paths. * * @warning This is a "fast" variant, meaning it assumes the buffer has enough * bits available. The caller is responsible for ensuring that at least `n` bits * are present in the accumulator or buffer before calling this function to * avoid undefined behavior or reading past valid memory. * * @param[in,out] br Pointer to the bit reader instance. * @param[in] n The number of bits to consume (must be <= 32, typically <= 24 * depending on implementation). * @return The value of the consumed bits as a 32-bit unsigned integer. */ static ZXC_ALWAYS_INLINE uint32_t zxc_br_consume_fast(zxc_bit_reader_t* RESTRICT br, const uint8_t n) { #if !defined(ZXC_DISABLE_SIMD) && defined(__BMI2__) && (defined(__x86_64__) || defined(_M_X64)) // BMI2 Optimization: _bzhi_u64(x, n) copies the lower n bits of x to dst and // clears the rest. It is equivalent to x & ((1ULL << n) - 1) but executes in // a single cycle without dependency chains. const uint32_t val = (uint32_t)_bzhi_u64(br->accum, n); #else const uint32_t val = (uint32_t)(br->accum & ((1ULL << n) - 1)); #endif br->accum >>= n; br->bits -= n; return val; } /** * @brief Reads a Prefix Varint encoded integer from a stream. * * This function decodes a 32-bit unsigned integer encoded in Prefix Varint format * from the provided byte stream. Unary prefix bits in the first byte determine * the total length (1-5 bytes). * * Format: * - 1 byte (0xxxxxxx): 7-bit payload (val < 2^7 = 128) * - 2 bytes (10xxxxxx): 14-bit payload (val < 2^14 = 16384) * - 3 bytes (110xxxxx): 21-bit payload (val < 2^21 = 2097152) * - 4 bytes (1110xxxx): 28-bit payload (val < 2^28 = 268435456) * - 5 bytes (11110xxx): 32-bit payload (full uint32_t range) * * @param[in,out] ptr Pointer to a pointer to the current position in the stream. * @param[in] end Pointer to the end of the readable stream (for bounds checking). * @return The decoded 32-bit integer, or 0 if reading would overflow bounds (safe default). */ static ZXC_ALWAYS_INLINE uint32_t zxc_read_varint(const uint8_t** ptr, const uint8_t* end) { const uint8_t* p = *ptr; // Bounds check: need at least 1 byte if (UNLIKELY(p >= end)) return 0; const uint32_t b0 = p[0]; // 1 Byte: 0xxxxxxx (7 bits) -> val < 128 (2^7) if (LIKELY(b0 < 0x80)) { *ptr = p + 1; return b0; } // 2 Bytes: 10xxxxxx xxxxxxxx (14 bits) -> val < 16384 (2^14) if (LIKELY(b0 < 0xC0)) { if (UNLIKELY(p + 1 >= end)) { *ptr = end; return 0; } *ptr = p + 2; return (b0 & 0x3F) | ((uint32_t)p[1] << 6); } // 3 Bytes: 110xxxxx xxxxxxxx xxxxxxxx (21 bits) -> val < 2097152 (2^21) if (LIKELY(b0 < 0xE0)) { if (UNLIKELY(p + 2 >= end)) { *ptr = end; return 0; } *ptr = p + 3; return (b0 & 0x1F) | ((uint32_t)p[1] << 5) | ((uint32_t)p[2] << 13); } // 4 Bytes: 1110xxxx ... (28 bits) -> val < 268435456 (2^28) if (UNLIKELY(b0 < 0xF0)) { if (UNLIKELY(p + 3 >= end)) { *ptr = end; return 0; } *ptr = p + 4; return (b0 & 0x0F) | ((uint32_t)p[1] << 4) | ((uint32_t)p[2] << 12) | ((uint32_t)p[3] << 20); } // 5 Bytes: 11110xxx ... (32 bits) -> val < 4294967296 (2^32) if (UNLIKELY(p + 4 >= end)) { *ptr = end; return 0; } *ptr = p + 5; return (b0 & 0x07) | ((uint32_t)p[1] << 3) | ((uint32_t)p[2] << 11) | ((uint32_t)p[3] << 19) | ((uint32_t)p[4] << 27); } /** * @brief Shuffle masks for overlapping copies with small offsets (0-15). * * Shared between ARM NEON and x86 SSSE3. Each row defines how to replicate * source bytes to fill 16 bytes when offset < 16. */ #if defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) || defined(ZXC_USE_AVX2) || \ defined(ZXC_USE_AVX512) /** * @brief Precomputed masks for handling overlapping data during decompression. * * This 16x16 lookup table contains 128-bit aligned masks used to efficiently * mask off or combine bytes when processing overlapping copy operations or * boundary conditions in the ZXC decompression algorithm. * * The alignment to 16 bytes ensures compatibility with SIMD instructions * (like SSE/AVX) for optimized memory operations. */ static const ZXC_ALIGN(16) uint8_t zxc_overlap_masks[16][16] = { {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, // off=0 (unused) {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, // off=1 (RLE handled separately) {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1}, // off=2 {0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0}, // off=3 {0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3}, // off=4 {0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0}, // off=5 {0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3}, // off=6 {0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1}, // off=7 {0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7}, // off=8 {0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6}, // off=9 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5}, // off=10 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0, 1, 2, 3, 4}, // off=11 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3}, // off=12 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0, 1, 2}, // off=13 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0, 1}, // off=14 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0} // off=15 }; #endif /** * @brief Copies 16 bytes from an overlapping source to the destination. * * This function is designed to handle memory copies where the source and * destination regions might overlap, specifically copying 16 bytes from * `dst - off` to `dst`. It is typically used in decompression routines * (like LZ77) where repeating a previous sequence is required. * * Handles NEON64, NEON32, SSSE3/AVX2 and generic scalar fallback. * * @param[out] dst Pointer to the destination buffer where bytes will be written. * @param[in] off The offset backwards from the destination pointer to read from. * (i.e., source address is `dst - off`). */ // codeql[cpp/unused-static-function] : False positive, used in DECODE_SEQ_SAFE/FAST macros static ZXC_ALWAYS_INLINE void zxc_copy_overlap16(uint8_t* dst, uint32_t off) { // off is always >= ZXC_LZ_OFFSET_BIAS by design (offset bias encoding: stored + // ZXC_LZ_OFFSET_BIAS) #if defined(ZXC_USE_NEON64) uint8x16_t mask = vld1q_u8(zxc_overlap_masks[off]); uint8x16_t src_data = vld1q_u8(dst - off); vst1q_u8(dst, vqtbl1q_u8(src_data, mask)); #elif defined(ZXC_USE_NEON32) uint8x8x2_t src_tbl; src_tbl.val[0] = vld1_u8(dst - off); src_tbl.val[1] = vld1_u8(dst - off + 8); uint8x8_t mask_lo = vld1_u8(zxc_overlap_masks[off]); uint8x8_t mask_hi = vld1_u8(zxc_overlap_masks[off] + 8); vst1_u8(dst, vtbl2_u8(src_tbl, mask_lo)); vst1_u8(dst + 8, vtbl2_u8(src_tbl, mask_hi)); #elif defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) __m128i mask = _mm_load_si128((const __m128i*)zxc_overlap_masks[off]); __m128i src_data = _mm_loadu_si128((const __m128i*)(dst - off)); _mm_storeu_si128((__m128i*)dst, _mm_shuffle_epi8(src_data, mask)); #else const uint8_t* src = dst - off; for (size_t i = 0; i < 16; i++) { dst[i] = src[i % off]; } #endif } #if defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) /** * @brief Computes the prefix sum of a 128-bit vector of 32-bit unsigned * integers using NEON intrinsics. * * This function calculates the running total of the elements in the input * vector `v`. If the input vector is `[a, b, c, d]`, the result will be `[a, * a+b, a+b+c, a+b+c+d]`. This operation is typically used for calculating * cumulative distributions or offsets in parallel. * * @param[in] v The input vector containing four 32-bit unsigned integers. * @return A uint32x4_t vector containing the prefix sums. */ static ZXC_ALWAYS_INLINE uint32x4_t zxc_neon_prefix_sum_u32(uint32x4_t v) { const uint32x4_t zero = vdupq_n_u32(0); // Create a vector of zeros // Rotate right by 1 element (shift 4 bytes) const uint32x4_t s1 = vreinterpretq_u32_u8(vextq_u8(vreinterpretq_u8_u32(zero), vreinterpretq_u8_u32(v), 12)); v = vaddq_u32(v, s1); // Add shifted version: [a, b, c, d] + [0, a, b, c] -> // [a, a+b, b+c, c+d] // Rotate right by 2 elements (shift 8 bytes) const uint32x4_t s2 = vreinterpretq_u32_u8(vextq_u8(vreinterpretq_u8_u32(zero), vreinterpretq_u8_u32(v), 8)); v = vaddq_u32(v, s2); // Add shifted version to complete prefix sum return v; } #endif #if defined(ZXC_USE_AVX2) /** * @brief Computes the prefix sum of 32-bit integers within a 256-bit vector. * * This function calculates the cumulative sum of the eight 32-bit integers * contained in the input vector `v`. * * Operation logic (conceptually): * out[0] = v[0] * out[1] = v[0] + v[1] * ... * out[7] = v[0] + v[1] + ... + v[7] * * @param[in] v The input 256-bit vector containing eight 32-bit integers. * @return A 256-bit vector containing the prefix sums of the input elements. */ // codeql[cpp/unused-static-function] : Used conditionally when ZXC_USE_AVX2 is defined static ZXC_ALWAYS_INLINE __m256i zxc_mm256_prefix_sum_epi32(__m256i v) { v = _mm256_add_epi32(v, _mm256_slli_si256(v, 4)); // Add value shifted by 1 element v = _mm256_add_epi32(v, _mm256_slli_si256(v, 8)); // Add value shifted by 2 elements // Use permute/shuffle to bridge the 128-bit lane gap __m256i v_bridge = _mm256_permute2x128_si256(v, v, 0x00); // Duplicate lower 128 to upper v_bridge = _mm256_shuffle_epi32(v_bridge, 0xFF); // Broadcast last element of lower 128 v_bridge = _mm256_blend_epi32(_mm256_setzero_si256(), v_bridge, 0xF0); // Only apply to upper lane return _mm256_add_epi32(v, v_bridge); // Add bridge value to upper lane } #endif #if defined(ZXC_USE_AVX512) /** * @brief Computes the prefix sum of 32-bit integers within a 512-bit vector. * * This function calculates the running sum of the 16 packed 32-bit integers * in the input vector `v`. * * For an input vector v = [x0, x1, x2, ... x15], the result will be: * [x0, x0+x1, x0+x1+x2, ... , sum(x0..x15)]. * * @note This function is forced inline for performance reasons. * * @param[in] v The input 512-bit vector containing sixteen 32-bit integers. * @return A 512-bit vector containing the prefix sums of the input elements. */ static ZXC_ALWAYS_INLINE __m512i zxc_mm512_prefix_sum_epi32(__m512i v) { __m512i t = _mm512_bslli_epi128(v, 4); // Shift left by 4 bytes (1 int) v = _mm512_add_epi32(v, t); // Add shifted value t = _mm512_bslli_epi128(v, 8); // Shift left by 8 bytes (2 ints) v = _mm512_add_epi32(v, t); // Add shifted value // Propagate sums across 128-bit lanes (sequential dependency) __m512i v_l0 = _mm512_shuffle_i32x4(v, v, 0x00); // Broadcast lane 0 v_l0 = _mm512_shuffle_epi32(v_l0, 0xFF); // Broadcast last element of lane 0 v = _mm512_mask_add_epi32(v, 0x00F0, v, v_l0); // Add to lane 1 only __m512i v_l1 = _mm512_shuffle_i32x4(v, v, 0x55); // Broadcast lane 1 v_l1 = _mm512_shuffle_epi32(v_l1, 0xFF); // Broadcast last element of lane 1 v = _mm512_mask_add_epi32(v, 0x0F00, v, v_l1); // Add to lane 2 only __m512i v_l2 = _mm512_shuffle_i32x4(v, v, 0xAA); // Broadcast lane 2 v_l2 = _mm512_shuffle_epi32(v_l2, 0xFF); // Broadcast last element of lane 2 v = _mm512_mask_add_epi32(v, 0xF000, v, v_l2); // Add to lane 3 only return v; } #endif /** * @brief Decodes a block of numerical data compressed with the ZXC format. * * This function reads a compressed numerical block from the source buffer, * parses the header to determine the number of values and encoding parameters, * and then decompresses the data into the destination buffer. * * **Algorithm Details:** * 1. **Header Parsing:** Reads the `zxc_num_header_t` to get the count of * values. * 2. **Bit Unpacking:** For each chunk of values, it initializes a bit reader. * - **Unrolling:** The main loop is unrolled 4x to minimize branch overhead * and maximize instruction throughput. * 3. **ZigZag Decoding:** Converts the unsigned unpacked value back to a signed * delta using `(n >> 1) ^ -(n & 1)`. * 4. **Delta Reconstruction:** Adds the signed delta to a `running_val` * accumulator to recover the original integer sequence. * * @param[in] src Pointer to the source buffer containing compressed data. * @param[in] src_size Size of the source buffer in bytes. * @param[out] dst Pointer to the destination buffer where decompressed data will be * written. * @param[in] dst_capacity Maximum capacity of the destination buffer in bytes. * * @return The number of bytes written to the destination buffer on success, * or a negative zxc_error_t code if an error occurs (e.g., buffer overflow, invalid header, * or malformed compressed stream). */ static int zxc_decode_block_num(const uint8_t* RESTRICT src, const size_t src_size, uint8_t* RESTRICT dst, const size_t dst_capacity) { zxc_num_header_t nh; if (UNLIKELY(zxc_read_num_header(src, src_size, &nh) != ZXC_OK)) return ZXC_ERROR_BAD_HEADER; size_t offset = ZXC_NUM_HEADER_BINARY_SIZE; uint8_t* d_ptr = dst; const uint8_t* const d_end = dst + dst_capacity; uint64_t vals_remaining = nh.n_values; uint32_t running_val = 0; ZXC_ALIGN(ZXC_CACHE_LINE_SIZE) uint32_t deltas[ZXC_NUM_DEC_BATCH]; while (vals_remaining > 0) { if (UNLIKELY(offset > src_size - ZXC_NUM_CHUNK_HEADER_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; const uint16_t nvals = zxc_le16(src + offset); const uint16_t bits = zxc_le16(src + offset + 2); const uint32_t psize = zxc_le32(src + offset + 12); // padding + nvals + bits offset += ZXC_NUM_CHUNK_HEADER_SIZE; if (UNLIKELY(nvals > vals_remaining || psize > src_size - offset || (size_t)(d_end - d_ptr) < (size_t)nvals * sizeof(uint32_t) || bits > (sizeof(uint32_t) * CHAR_BIT))) return ZXC_ERROR_CORRUPT_DATA; zxc_bit_reader_t br; zxc_br_init(&br, src + offset, psize); size_t i = 0; for (; i + ZXC_NUM_DEC_BATCH <= nvals; i += ZXC_NUM_DEC_BATCH) { for (int k = 0; k < ZXC_NUM_DEC_BATCH; k += 4) { zxc_br_ensure(&br, bits); deltas[k + 0] = zxc_zigzag_decode(zxc_br_consume_fast(&br, (uint8_t)bits)); zxc_br_ensure(&br, bits); deltas[k + 1] = zxc_zigzag_decode(zxc_br_consume_fast(&br, (uint8_t)bits)); zxc_br_ensure(&br, bits); deltas[k + 2] = zxc_zigzag_decode(zxc_br_consume_fast(&br, (uint8_t)bits)); zxc_br_ensure(&br, bits); deltas[k + 3] = zxc_zigzag_decode(zxc_br_consume_fast(&br, (uint8_t)bits)); } uint32_t* batch_dst = (uint32_t*)d_ptr; #if defined(ZXC_USE_AVX512) __m512i v_run = _mm512_set1_epi32(running_val); // Broadcast initial running total for (int k = 0; k < ZXC_NUM_DEC_BATCH; k += 16) { __m512i v_deltas = _mm512_load_si512((void*)&deltas[k]); // Load 16 deltas __m512i v_sum = zxc_mm512_prefix_sum_epi32(v_deltas); // Compute local prefix sums v_sum = _mm512_add_epi32(v_sum, v_run); // Add base running total _mm512_storeu_si512((void*)&batch_dst[k], v_sum); // Store decoded values // Broadcast 15th element of v_sum to v_run directly within ZMM registers // 1. Align upper 128-bit lane down to all lanes __m512i v_last128 = _mm512_shuffle_i32x4(v_sum, v_sum, 0xFF); // 2. Broadcast the 3rd element of those lanes v_run = _mm512_shuffle_epi32(v_last128, 0xFF); } // Extract final running_val back to GPR for scalar fallback running_val = (uint32_t)_mm_cvtsi128_si32(_mm512_castsi512_si128(v_run)); #elif defined(ZXC_USE_AVX2) __m256i v_run = _mm256_set1_epi32(running_val); // Broadcast initial running total for (int k = 0; k < ZXC_NUM_DEC_BATCH; k += 8) { __m256i v_deltas = _mm256_load_si256((const __m256i*)&deltas[k]); // Load 8 deltas __m256i v_sum = zxc_mm256_prefix_sum_epi32(v_deltas); // Compute local prefix sums v_sum = _mm256_add_epi32(v_sum, v_run); // Add base _mm256_storeu_si256((__m256i*)&batch_dst[k], v_sum); // Store decoded values // Compute v_run directly from vector register without memory readback // Duplicate upper 128-bits into both lanes __m256i last_val = _mm256_permute2x128_si256(v_sum, v_sum, 0x11); // Broadcast 4th element to all elements v_run = _mm256_shuffle_epi32(last_val, 0xFF); } // Extract final running_val back to GPR for scalar fallback running_val = (uint32_t)_mm_cvtsi128_si32(_mm256_castsi256_si128(v_run)); #elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) uint32x4_t v_run = vdupq_n_u32(running_val); // Broadcast running total for (int k = 0; k < ZXC_NUM_DEC_BATCH; k += 4) { uint32x4_t v_deltas = vld1q_u32(&deltas[k]); // Load 4 deltas uint32x4_t v_sum = zxc_neon_prefix_sum_u32(v_deltas); // Compute local prefix sums v_sum = vaddq_u32(v_sum, v_run); // Add base vst1q_u32(&batch_dst[k], v_sum); // Store decoded values #if defined(ZXC_USE_NEON64) v_run = vdupq_laneq_u32(v_sum, 3); // Update vector directly (no GPR transit) #else running_val = vgetq_lane_u32(v_sum, 3); // Extract last element v_run = vdupq_n_u32(running_val); // Update vector for next iter #endif } #if defined(ZXC_USE_NEON64) running_val = vgetq_lane_u32(v_run, 0); // Extract once at the end of the batch #endif #else for (int k = 0; k < ZXC_NUM_DEC_BATCH; k++) { running_val += deltas[k]; #ifdef ZXC_BIG_ENDIAN zxc_store_le32(&batch_dst[k], running_val); #else batch_dst[k] = running_val; #endif } #endif d_ptr += ZXC_NUM_DEC_BATCH * sizeof(uint32_t); } for (; i < nvals; i++) { zxc_br_ensure(&br, bits); const uint32_t delta = zxc_zigzag_decode(zxc_br_consume_fast(&br, (uint8_t)bits)); running_val += delta; zxc_store_le32(d_ptr, running_val); d_ptr += sizeof(uint32_t); } offset += psize; vals_remaining -= nvals; } return (int)(d_ptr - dst); } /* ========================================================================== * Shared decode macros for the GLO and GHI decoders (fast + safe variants). * Defined at file scope to avoid four identical copies inside each function. * They reference the local names l_ptr, d_ptr, written that every call site * has in scope. #undef-ed at the end of the last consumer. * ========================================================================== */ // Copy literals from l_ptr to d_ptr using 32-byte wild copies #define DECODE_COPY_LITERALS(ll) \ do { \ const uint8_t* src_lit = l_ptr; \ uint8_t* dst_lit = d_ptr; \ zxc_copy32(dst_lit, src_lit); \ if (UNLIKELY(ll > ZXC_PAD_SIZE)) { \ dst_lit += ZXC_PAD_SIZE; \ src_lit += ZXC_PAD_SIZE; \ size_t rem = ll - ZXC_PAD_SIZE; \ while (rem > ZXC_PAD_SIZE) { \ zxc_copy32(dst_lit, src_lit); \ dst_lit += ZXC_PAD_SIZE; \ src_lit += ZXC_PAD_SIZE; \ rem -= ZXC_PAD_SIZE; \ } \ zxc_copy32(dst_lit, src_lit); \ } \ l_ptr += ll; \ d_ptr += ll; \ } while (0) // Copy match from d_ptr - off to d_ptr, handling overlap cases #define DECODE_COPY_MATCH(ml, off) \ do { \ const uint8_t* match_src = d_ptr - off; \ if (LIKELY(off >= ZXC_PAD_SIZE)) { \ zxc_copy32(d_ptr, match_src); \ if (UNLIKELY(ml > ZXC_PAD_SIZE)) { \ uint8_t* out = d_ptr + ZXC_PAD_SIZE; \ const uint8_t* ref = match_src + ZXC_PAD_SIZE; \ size_t rem = ml - ZXC_PAD_SIZE; \ while (rem > ZXC_PAD_SIZE) { \ zxc_copy32(out, ref); \ out += ZXC_PAD_SIZE; \ ref += ZXC_PAD_SIZE; \ rem -= ZXC_PAD_SIZE; \ } \ zxc_copy32(out, ref); \ } \ d_ptr += ml; \ } else if (off >= (ZXC_PAD_SIZE / 2)) { \ zxc_copy16(d_ptr, match_src); \ if (UNLIKELY(ml > (ZXC_PAD_SIZE / 2))) { \ uint8_t* out = d_ptr + (ZXC_PAD_SIZE / 2); \ const uint8_t* ref = match_src + (ZXC_PAD_SIZE / 2); \ size_t rem = ml - (ZXC_PAD_SIZE / 2); \ while (rem > (ZXC_PAD_SIZE / 2)) { \ zxc_copy16(out, ref); \ out += (ZXC_PAD_SIZE / 2); \ ref += (ZXC_PAD_SIZE / 2); \ rem -= (ZXC_PAD_SIZE / 2); \ } \ zxc_copy16(out, ref); \ } \ d_ptr += ml; \ } else if (off == 1) { \ ZXC_MEMSET(d_ptr, match_src[0], ml); \ d_ptr += ml; \ } else { \ size_t copied = 0; \ while (copied < ml) { \ zxc_copy_overlap16(d_ptr + copied, off); \ copied += (ZXC_PAD_SIZE / 2); \ } \ d_ptr += ml; \ } \ } while (0) // SAFE version: validates offset against written bytes #define DECODE_SEQ_SAFE(ll, ml, off) \ do { \ DECODE_COPY_LITERALS(ll); \ written += ll; \ if (UNLIKELY(off > written)) return ZXC_ERROR_BAD_OFFSET; \ DECODE_COPY_MATCH(ml, off); \ written += ml; \ } while (0) // FAST version: no offset validation (for use after written >= 256 or 65536) #define DECODE_SEQ_FAST(ll, ml, off) \ do { \ DECODE_COPY_LITERALS(ll); \ DECODE_COPY_MATCH(ml, off); \ } while (0) /** * @brief Decodes a General Low (GLO) format compressed block. * * This function handles the decoding of a compressed block formatted with the * internal GLO structure. The decompressed size is derived from Section * Descriptors within the compressed payload. * * @param[in,out] ctx Pointer to the compression context (`zxc_cctx_t`) containing * @param[in] src Pointer to the source buffer containing compressed data. * @param[in] src_size Size of the source buffer in bytes. * @param[out] dst Pointer to the destination buffer for decompressed data. * @param[in] dst_capacity Maximum capacity of the destination buffer. * * @return The number of bytes written to the destination buffer on success, or * a negative zxc_error_t code on failure (e.g., invalid header, buffer overflow, or corrupted * data). */ /** * @brief Unified GLO decoder body shared by the fast and safe variants. * * @p safe must be a compile-time constant (0 or 1). The two 4x-unrolled loops * are duplicated verbatim inside @c if(safe)/else branches so each variant * keeps single-assignment @c const save pointers. After constant propagation * only one branch survives per wrapper, yielding codegen equivalent to the * hand-written pair. */ static ZXC_ALWAYS_INLINE int zxc_decode_block_glo_impl(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_size, uint8_t* RESTRICT dst, const size_t dst_capacity, const int safe) { zxc_gnr_header_t gh; zxc_section_desc_t desc[ZXC_GLO_SECTIONS]; if (UNLIKELY(zxc_read_glo_header_and_desc(src, src_size, &gh, desc) != ZXC_OK)) return ZXC_ERROR_BAD_HEADER; const uint8_t* p_data = src + ZXC_GLO_HEADER_BINARY_SIZE + ZXC_GLO_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; const uint8_t* p_curr = p_data; // --- Literal Stream Setup --- const uint8_t* l_ptr; const uint8_t* l_end; uint8_t* rle_buf = NULL; size_t lit_stream_size = (size_t)(desc[0].sizes & ZXC_SECTION_SIZE_MASK); if (gh.enc_lit == ZXC_SECTION_ENCODING_HUFFMAN) { const size_t required_size = (size_t)(desc[0].sizes >> 32); if (UNLIKELY(lit_stream_size > (size_t)(src + src_size - p_curr))) return ZXC_ERROR_CORRUPT_DATA; if (required_size == 0) { l_ptr = p_curr; l_end = p_curr; } else { if (UNLIKELY(required_size > dst_capacity || required_size > SIZE_MAX - ZXC_PAD_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; const size_t alloc_size = required_size + ZXC_PAD_SIZE; if (UNLIKELY(ctx->lit_buffer_cap < alloc_size)) { uint8_t* new_buf = (uint8_t*)realloc(ctx->lit_buffer, alloc_size); if (UNLIKELY(!new_buf)) { free(ctx->lit_buffer); ctx->lit_buffer = NULL; ctx->lit_buffer_cap = 0; return ZXC_ERROR_MEMORY; } ctx->lit_buffer = new_buf; ctx->lit_buffer_cap = alloc_size; } const int rc = zxc_huf_decode_section(p_curr, lit_stream_size, ctx->lit_buffer, required_size); if (UNLIKELY(rc != ZXC_OK)) return rc; l_ptr = ctx->lit_buffer; l_end = ctx->lit_buffer + required_size; } } else if (gh.enc_lit == ZXC_SECTION_ENCODING_RLE) { const size_t required_size = (size_t)(desc[0].sizes >> 32); if (required_size > 0) { if (UNLIKELY(required_size > dst_capacity || required_size > SIZE_MAX - ZXC_PAD_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; const size_t alloc_size = required_size + ZXC_PAD_SIZE; if (UNLIKELY(ctx->lit_buffer_cap < alloc_size)) { uint8_t* new_buf = (uint8_t*)realloc(ctx->lit_buffer, alloc_size); if (UNLIKELY(!new_buf)) { free(ctx->lit_buffer); ctx->lit_buffer = NULL; ctx->lit_buffer_cap = 0; return ZXC_ERROR_MEMORY; } ctx->lit_buffer = new_buf; ctx->lit_buffer_cap = alloc_size; } rle_buf = ctx->lit_buffer; if (UNLIKELY(!rle_buf || lit_stream_size > (size_t)(src + src_size - p_curr))) return ZXC_ERROR_CORRUPT_DATA; const uint8_t* r_ptr = p_curr; const uint8_t* r_end = r_ptr + lit_stream_size; uint8_t* w_ptr = rle_buf; const uint8_t* const w_end = rle_buf + required_size; while (r_ptr < r_end && w_ptr < w_end) { uint8_t token = *r_ptr++; if (LIKELY(!(token & ZXC_LIT_RLE_FLAG))) { // Raw copy (most common path): use ZXC_PAD_SIZE-byte wild copies // token is 7-bit (0-127), so len is 1-128 bytes const uint32_t len = (uint32_t)token + 1; if (UNLIKELY(w_ptr + len > w_end || r_ptr + len > r_end)) return ZXC_ERROR_CORRUPT_DATA; // Destination has ZXC_PAD_SIZE bytes of safe overrun space. // Source may not - check before wild copy. // Fast path: source has ZXC_PAD_SIZE-byte read headroom (most common) if (LIKELY(r_ptr + ZXC_PAD_SIZE <= r_end)) { // Single 32-byte copy covers len <= ZXC_PAD_SIZE (most tokens) zxc_copy32(w_ptr, r_ptr); if (UNLIKELY(len > ZXC_PAD_SIZE)) { // Unroll: max len=128, so max 4 copies total // Use unconditional stores with overlap - faster than branches if (len <= 2 * ZXC_PAD_SIZE) { zxc_copy32(w_ptr + len - ZXC_PAD_SIZE, r_ptr + len - ZXC_PAD_SIZE); } else if (len <= 3 * ZXC_PAD_SIZE) { zxc_copy32(w_ptr + ZXC_PAD_SIZE, r_ptr + ZXC_PAD_SIZE); zxc_copy32(w_ptr + len - ZXC_PAD_SIZE, r_ptr + len - ZXC_PAD_SIZE); } else { zxc_copy32(w_ptr + ZXC_PAD_SIZE, r_ptr + ZXC_PAD_SIZE); zxc_copy32(w_ptr + 2 * ZXC_PAD_SIZE, r_ptr + 2 * ZXC_PAD_SIZE); zxc_copy32(w_ptr + len - ZXC_PAD_SIZE, r_ptr + len - ZXC_PAD_SIZE); } } } else { // Near end of source: safe copy (rare cold path) ZXC_MEMCPY(w_ptr, r_ptr, len); } w_ptr += len; r_ptr += len; } else { // RLE run: fill with single byte const uint32_t len = (token & ZXC_LIT_LEN_MASK) + 4; if (UNLIKELY(w_ptr + len > w_end || r_ptr >= r_end)) return ZXC_ERROR_CORRUPT_DATA; ZXC_MEMSET(w_ptr, *r_ptr++, len); w_ptr += len; } } if (UNLIKELY(w_ptr != w_end)) return ZXC_ERROR_CORRUPT_DATA; l_ptr = rle_buf; l_end = rle_buf + required_size; } else { l_ptr = p_curr; l_end = p_curr; } } else if (gh.enc_lit == ZXC_SECTION_ENCODING_RAW) { l_ptr = p_curr; l_end = p_curr + lit_stream_size; } else { return ZXC_ERROR_CORRUPT_DATA; } p_curr += lit_stream_size; // --- Stream Pointers & Validation --- const size_t sz_tokens = (size_t)(desc[1].sizes & ZXC_SECTION_SIZE_MASK); const size_t sz_offsets = (size_t)(desc[2].sizes & ZXC_SECTION_SIZE_MASK); const size_t sz_extras = (size_t)(desc[3].sizes & ZXC_SECTION_SIZE_MASK); // Validate stream sizes match sequence count (early rejection of malformed data) const uint64_t expected_off_size = (gh.enc_off == 1) ? (uint64_t)gh.n_sequences : (uint64_t)gh.n_sequences * 2; const uint8_t* t_ptr = p_curr; const uint8_t* o_ptr = t_ptr + sz_tokens; const uint8_t* e_ptr = o_ptr + sz_offsets; const uint8_t* const e_end = e_ptr + sz_extras; // For vbyte overflow detection // Validate streams don't overflow source buffer + // Validate stream sizes match sequence count (early rejection of malformed data) if (UNLIKELY((e_end != src + src_size) || sz_tokens < gh.n_sequences || (uint64_t)sz_offsets < expected_off_size)) return ZXC_ERROR_CORRUPT_DATA; uint8_t* d_ptr = dst; const uint8_t* const d_end = dst + dst_capacity; // Destination safe margin for 4x loop: max output without varint extension. // ll_max = 14, ml_max = 14 + 5 = 19, per-seq = 33, 4x = 132. // Add ZXC_PAD_SIZE (32) for the wild zxc_copy32 overshoot + 4 safety = 168. const uint8_t* const d_end_safe = d_end - (132 + ZXC_PAD_SIZE + 4); // Literal stream safe threshold for 4x-unrolled loops. // Without varint extension, max ll per sequence = ZXC_TOKEN_LL_MASK - 1 = 14. // For 4 sequences: 4 * 14 = 56. With this margin, l_ptr checks are only needed // on the cold varint path, keeping the hot path free of l_ptr overhead. const size_t glo_sz_lit = (size_t)(l_end - l_ptr); const size_t glo_margin_4x = 4 * (ZXC_TOKEN_LL_MASK - 1); // 56 const size_t glo_margin_1x = ZXC_TOKEN_LL_MASK - 1; // 14 const uint8_t* const l_end_safe_4x = (glo_sz_lit > glo_margin_4x) ? l_end - glo_margin_4x : l_ptr; const uint8_t* const l_end_safe_1x = (glo_sz_lit > glo_margin_1x) ? l_end - glo_margin_1x : l_ptr; uint32_t n_seq = gh.n_sequences; // Track bytes written for offset validation // For 1-byte offsets (enc_off==1): validate until 256 bytes written (max 8-bit offset) // For 2-byte offsets (enc_off==0): validate until 65536 bytes written (max 16-bit offset) // After threshold, all offsets are guaranteed valid (can't exceed written bytes) size_t written = 0; // --- SAFE Loop: offset validation until threshold (4x unroll) --- // For 1-byte offsets: bounds check until 256 bytes written // For 2-byte offsets: bounds check until 65536 bytes written const size_t bounds_threshold = (gh.enc_off == 1) ? (1U << 8) : (1U << 16); if (safe) { /* SAFE variant: save per-batch state so overflow can rollback. */ while (n_seq >= 4 && d_ptr < d_end_safe && l_ptr < l_end_safe_4x && written < bounds_threshold) { const uint8_t* const t_save = t_ptr; const uint8_t* const o_save = o_ptr; const uint8_t* const e_save = e_ptr; uint8_t* const d_save = d_ptr; const uint8_t* const l_save = l_ptr; const size_t w_save = written; uint32_t tokens = zxc_le32(t_ptr); t_ptr += 4; uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, off4 = ZXC_LZ_OFFSET_BIAS; if (gh.enc_off == 1) { uint32_t offsets = zxc_le32(o_ptr); o_ptr += 4; off1 += offsets & 0xFF; off2 += (offsets >> 8) & 0xFF; off3 += (offsets >> 16) & 0xFF; off4 += (offsets >> 24) & 0xFF; } else { uint64_t offsets = zxc_le64(o_ptr); o_ptr += 8; off1 += (uint32_t)(offsets & 0xFFFF); off2 += (uint32_t)((offsets >> 16) & 0xFFFF); off3 += (uint32_t)((offsets >> 32) & 0xFFFF); off4 += (uint32_t)((offsets >> 48) & 0xFFFF); } uint32_t ll1 = (tokens & 0x0F0) >> 4; uint32_t ml1 = (tokens & 0x00F); if (UNLIKELY(ll1 == ZXC_TOKEN_LL_MASK)) { ll1 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(l_ptr + ll1 > l_end || d_ptr + ll1 + ZXC_PAD_SIZE > d_end)) goto rollback_safe_4x; } if (UNLIKELY(ml1 == ZXC_TOKEN_ML_MASK)) { ml1 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(d_ptr + ll1 + ml1 + ZXC_LZ_MIN_MATCH_LEN + ZXC_PAD_SIZE > d_end)) goto rollback_safe_4x; } ml1 += ZXC_LZ_MIN_MATCH_LEN; DECODE_SEQ_SAFE(ll1, ml1, off1); uint32_t ll2 = (tokens & 0x0F000) >> 12; uint32_t ml2 = (tokens & 0x00F00) >> 8; if (UNLIKELY(ll2 == ZXC_TOKEN_LL_MASK)) { ll2 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(l_ptr + ll2 > l_end || d_ptr + ll2 + ZXC_PAD_SIZE > d_end)) goto rollback_safe_4x; } if (UNLIKELY(ml2 == ZXC_TOKEN_ML_MASK)) { ml2 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(d_ptr + ll2 + ml2 + ZXC_LZ_MIN_MATCH_LEN + ZXC_PAD_SIZE > d_end)) goto rollback_safe_4x; } ml2 += ZXC_LZ_MIN_MATCH_LEN; DECODE_SEQ_SAFE(ll2, ml2, off2); uint32_t ll3 = (tokens & 0x0F00000) >> 20; uint32_t ml3 = (tokens & 0x00F0000) >> 16; if (UNLIKELY(ll3 == ZXC_TOKEN_LL_MASK)) { ll3 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(l_ptr + ll3 > l_end || d_ptr + ll3 + ZXC_PAD_SIZE > d_end)) goto rollback_safe_4x; } if (UNLIKELY(ml3 == ZXC_TOKEN_ML_MASK)) { ml3 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(d_ptr + ll3 + ml3 + ZXC_LZ_MIN_MATCH_LEN + ZXC_PAD_SIZE > d_end)) goto rollback_safe_4x; } ml3 += ZXC_LZ_MIN_MATCH_LEN; DECODE_SEQ_SAFE(ll3, ml3, off3); uint32_t ll4 = (tokens >> 28); uint32_t ml4 = (tokens >> 24) & 0x0F; if (UNLIKELY(ll4 == ZXC_TOKEN_LL_MASK)) { ll4 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(l_ptr + ll4 > l_end || d_ptr + ll4 + ZXC_PAD_SIZE > d_end)) goto rollback_safe_4x; } if (UNLIKELY(ml4 == ZXC_TOKEN_ML_MASK)) { ml4 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(d_ptr + ll4 + ml4 + ZXC_LZ_MIN_MATCH_LEN + ZXC_PAD_SIZE > d_end)) goto rollback_safe_4x; } ml4 += ZXC_LZ_MIN_MATCH_LEN; DECODE_SEQ_SAFE(ll4, ml4, off4); n_seq -= 4; continue; rollback_safe_4x: t_ptr = t_save; o_ptr = o_save; e_ptr = e_save; d_ptr = d_save; l_ptr = l_save; written = w_save; break; } } else { while (n_seq >= 4 && d_ptr < d_end_safe && l_ptr < l_end_safe_4x && written < bounds_threshold) { uint32_t tokens = zxc_le32(t_ptr); t_ptr += 4; uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, off4 = ZXC_LZ_OFFSET_BIAS; if (gh.enc_off == 1) { // Read 4 x 1-byte offsets uint32_t offsets = zxc_le32(o_ptr); o_ptr += 4; off1 += offsets & 0xFF; off2 += (offsets >> 8) & 0xFF; off3 += (offsets >> 16) & 0xFF; off4 += (offsets >> 24) & 0xFF; } else { // Read 4 x 2-byte offsets uint64_t offsets = zxc_le64(o_ptr); o_ptr += 8; off1 += (uint32_t)(offsets & 0xFFFF); off2 += (uint32_t)((offsets >> 16) & 0xFFFF); off3 += (uint32_t)((offsets >> 32) & 0xFFFF); off4 += (uint32_t)((offsets >> 48) & 0xFFFF); } uint32_t ll1 = (tokens & 0x0F0) >> 4; uint32_t ml1 = (tokens & 0x00F); if (UNLIKELY(ll1 == ZXC_TOKEN_LL_MASK)) { ll1 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(l_ptr + ll1 > l_end || d_ptr + ll1 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } if (UNLIKELY(ml1 == ZXC_TOKEN_ML_MASK)) { ml1 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(d_ptr + ll1 + ml1 + ZXC_LZ_MIN_MATCH_LEN + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } ml1 += ZXC_LZ_MIN_MATCH_LEN; DECODE_SEQ_SAFE(ll1, ml1, off1); uint32_t ll2 = (tokens & 0x0F000) >> 12; uint32_t ml2 = (tokens & 0x00F00) >> 8; if (UNLIKELY(ll2 == ZXC_TOKEN_LL_MASK)) { ll2 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(l_ptr + ll2 > l_end || d_ptr + ll2 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } if (UNLIKELY(ml2 == ZXC_TOKEN_ML_MASK)) { ml2 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(d_ptr + ll2 + ml2 + ZXC_LZ_MIN_MATCH_LEN + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } ml2 += ZXC_LZ_MIN_MATCH_LEN; DECODE_SEQ_SAFE(ll2, ml2, off2); uint32_t ll3 = (tokens & 0x0F00000) >> 20; uint32_t ml3 = (tokens & 0x00F0000) >> 16; if (UNLIKELY(ll3 == ZXC_TOKEN_LL_MASK)) { ll3 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(l_ptr + ll3 > l_end || d_ptr + ll3 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } if (UNLIKELY(ml3 == ZXC_TOKEN_ML_MASK)) { ml3 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(d_ptr + ll3 + ml3 + ZXC_LZ_MIN_MATCH_LEN + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } ml3 += ZXC_LZ_MIN_MATCH_LEN; DECODE_SEQ_SAFE(ll3, ml3, off3); uint32_t ll4 = (tokens >> 28); uint32_t ml4 = (tokens >> 24) & 0x0F; if (UNLIKELY(ll4 == ZXC_TOKEN_LL_MASK)) { ll4 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(l_ptr + ll4 > l_end || d_ptr + ll4 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } if (UNLIKELY(ml4 == ZXC_TOKEN_ML_MASK)) { ml4 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(d_ptr + ll4 + ml4 + ZXC_LZ_MIN_MATCH_LEN + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } ml4 += ZXC_LZ_MIN_MATCH_LEN; DECODE_SEQ_SAFE(ll4, ml4, off4); n_seq -= 4; } } // --- FAST Loop: After threshold, no offset validation needed (4x unroll) --- if (safe) { while (n_seq >= 4 && d_ptr < d_end_safe && l_ptr < l_end_safe_4x) { const uint8_t* const t_save = t_ptr; const uint8_t* const o_save = o_ptr; const uint8_t* const e_save = e_ptr; uint8_t* const d_save = d_ptr; const uint8_t* const l_save = l_ptr; uint32_t tokens = zxc_le32(t_ptr); t_ptr += 4; uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, off4 = ZXC_LZ_OFFSET_BIAS; if (gh.enc_off == 1) { uint32_t offsets = zxc_le32(o_ptr); o_ptr += 4; off1 += offsets & 0xFF; off2 += (offsets >> 8) & 0xFF; off3 += (offsets >> 16) & 0xFF; off4 += (offsets >> 24) & 0xFF; } else { uint64_t offsets = zxc_le64(o_ptr); o_ptr += 8; off1 += (uint32_t)(offsets & 0xFFFF); off2 += (uint32_t)((offsets >> 16) & 0xFFFF); off3 += (uint32_t)((offsets >> 32) & 0xFFFF); off4 += (uint32_t)((offsets >> 48) & 0xFFFF); } uint32_t ll1 = (tokens & 0x0F0) >> 4; uint32_t ml1 = (tokens & 0x00F); if (UNLIKELY(ll1 == ZXC_TOKEN_LL_MASK)) { ll1 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(l_ptr + ll1 > l_end || d_ptr + ll1 + ZXC_PAD_SIZE > d_end)) goto rollback_fast_4x; } if (UNLIKELY(ml1 == ZXC_TOKEN_ML_MASK)) { ml1 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(d_ptr + ll1 + ml1 + ZXC_LZ_MIN_MATCH_LEN + ZXC_PAD_SIZE > d_end)) goto rollback_fast_4x; } ml1 += ZXC_LZ_MIN_MATCH_LEN; DECODE_SEQ_FAST(ll1, ml1, off1); uint32_t ll2 = (tokens & 0x0F000) >> 12; uint32_t ml2 = (tokens & 0x00F00) >> 8; if (UNLIKELY(ll2 == ZXC_TOKEN_LL_MASK)) { ll2 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(l_ptr + ll2 > l_end || d_ptr + ll2 + ZXC_PAD_SIZE > d_end)) goto rollback_fast_4x; } if (UNLIKELY(ml2 == ZXC_TOKEN_ML_MASK)) { ml2 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(d_ptr + ll2 + ml2 + ZXC_LZ_MIN_MATCH_LEN + ZXC_PAD_SIZE > d_end)) goto rollback_fast_4x; } ml2 += ZXC_LZ_MIN_MATCH_LEN; DECODE_SEQ_FAST(ll2, ml2, off2); uint32_t ll3 = (tokens & 0x0F00000) >> 20; uint32_t ml3 = (tokens & 0x00F0000) >> 16; if (UNLIKELY(ll3 == ZXC_TOKEN_LL_MASK)) { ll3 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(l_ptr + ll3 > l_end || d_ptr + ll3 + ZXC_PAD_SIZE > d_end)) goto rollback_fast_4x; } if (UNLIKELY(ml3 == ZXC_TOKEN_ML_MASK)) { ml3 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(d_ptr + ll3 + ml3 + ZXC_LZ_MIN_MATCH_LEN + ZXC_PAD_SIZE > d_end)) goto rollback_fast_4x; } ml3 += ZXC_LZ_MIN_MATCH_LEN; DECODE_SEQ_FAST(ll3, ml3, off3); uint32_t ll4 = (tokens >> 28); uint32_t ml4 = (tokens >> 24) & 0x0F; if (UNLIKELY(ll4 == ZXC_TOKEN_LL_MASK)) { ll4 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(l_ptr + ll4 > l_end || d_ptr + ll4 + ZXC_PAD_SIZE > d_end)) goto rollback_fast_4x; } if (UNLIKELY(ml4 == ZXC_TOKEN_ML_MASK)) { ml4 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(d_ptr + ll4 + ml4 + ZXC_LZ_MIN_MATCH_LEN + ZXC_PAD_SIZE > d_end)) goto rollback_fast_4x; } ml4 += ZXC_LZ_MIN_MATCH_LEN; DECODE_SEQ_FAST(ll4, ml4, off4); n_seq -= 4; continue; rollback_fast_4x: t_ptr = t_save; o_ptr = o_save; e_ptr = e_save; d_ptr = d_save; l_ptr = l_save; break; } } else { while (n_seq >= 4 && d_ptr < d_end_safe && l_ptr < l_end_safe_4x) { uint32_t tokens = zxc_le32(t_ptr); t_ptr += 4; uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, off4 = ZXC_LZ_OFFSET_BIAS; if (gh.enc_off == 1) { // Read 4 x 1-byte offsets uint32_t offsets = zxc_le32(o_ptr); o_ptr += 4; off1 += offsets & 0xFF; off2 += (offsets >> 8) & 0xFF; off3 += (offsets >> 16) & 0xFF; off4 += (offsets >> 24) & 0xFF; } else { // Read 4 x 2-byte offsets uint64_t offsets = zxc_le64(o_ptr); o_ptr += 8; off1 += (uint32_t)(offsets & 0xFFFF); off2 += (uint32_t)((offsets >> 16) & 0xFFFF); off3 += (uint32_t)((offsets >> 32) & 0xFFFF); off4 += (uint32_t)((offsets >> 48) & 0xFFFF); } uint32_t ll1 = (tokens & 0x0F0) >> 4; uint32_t ml1 = (tokens & 0x00F); if (UNLIKELY(ll1 == ZXC_TOKEN_LL_MASK)) { ll1 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(l_ptr + ll1 > l_end || d_ptr + ll1 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } if (UNLIKELY(ml1 == ZXC_TOKEN_ML_MASK)) { ml1 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(d_ptr + ll1 + ml1 + ZXC_LZ_MIN_MATCH_LEN + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } ml1 += ZXC_LZ_MIN_MATCH_LEN; DECODE_SEQ_FAST(ll1, ml1, off1); uint32_t ll2 = (tokens & 0x0F000) >> 12; uint32_t ml2 = (tokens & 0x00F00) >> 8; if (UNLIKELY(ll2 == ZXC_TOKEN_LL_MASK)) { ll2 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(l_ptr + ll2 > l_end || d_ptr + ll2 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } if (UNLIKELY(ml2 == ZXC_TOKEN_ML_MASK)) { ml2 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(d_ptr + ll2 + ml2 + ZXC_LZ_MIN_MATCH_LEN + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } ml2 += ZXC_LZ_MIN_MATCH_LEN; DECODE_SEQ_FAST(ll2, ml2, off2); uint32_t ll3 = (tokens & 0x0F00000) >> 20; uint32_t ml3 = (tokens & 0x00F0000) >> 16; if (UNLIKELY(ll3 == ZXC_TOKEN_LL_MASK)) { ll3 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(l_ptr + ll3 > l_end || d_ptr + ll3 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } if (UNLIKELY(ml3 == ZXC_TOKEN_ML_MASK)) { ml3 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(d_ptr + ll3 + ml3 + ZXC_LZ_MIN_MATCH_LEN + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } ml3 += ZXC_LZ_MIN_MATCH_LEN; DECODE_SEQ_FAST(ll3, ml3, off3); uint32_t ll4 = (tokens >> 28); uint32_t ml4 = (tokens >> 24) & 0x0F; if (UNLIKELY(ll4 == ZXC_TOKEN_LL_MASK)) { ll4 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(l_ptr + ll4 > l_end || d_ptr + ll4 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } if (UNLIKELY(ml4 == ZXC_TOKEN_ML_MASK)) { ml4 += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(d_ptr + ll4 + ml4 + ZXC_LZ_MIN_MATCH_LEN + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } ml4 += ZXC_LZ_MIN_MATCH_LEN; DECODE_SEQ_FAST(ll4, ml4, off4); n_seq -= 4; } } // Validate vbyte reads didn't overflow if (UNLIKELY(e_ptr > e_end)) return ZXC_ERROR_CORRUPT_DATA; // --- Remaining 1 sequence (Fast Path) --- while (n_seq > 0 && d_ptr < d_end_safe && l_ptr < l_end_safe_1x) { // Save pointers before reading (in case we need to fall back to Safe Path) const uint8_t* t_save = t_ptr; const uint8_t* o_save = o_ptr; const uint8_t* e_save = e_ptr; uint8_t token = *t_ptr++; uint32_t ll = token >> ZXC_TOKEN_LIT_BITS; uint32_t ml = token & ZXC_TOKEN_ML_MASK; uint32_t offset = ZXC_LZ_OFFSET_BIAS; if (gh.enc_off == 1) { offset += *o_ptr++; // 1-byte offset (biased) } else { offset += zxc_le16(o_ptr); // 2-byte offset (biased) o_ptr += 2; } if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { ll += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(l_ptr + ll > l_end)) { t_ptr = t_save; o_ptr = o_save; e_ptr = e_save; break; } } if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) ml += zxc_read_varint(&e_ptr, e_end); ml += ZXC_LZ_MIN_MATCH_LEN; // Check bounds before wild copies - if too close to end, fall back to Safe Path if (UNLIKELY(d_ptr + ll + ml + ZXC_PAD_SIZE > d_end)) { // Restore pointers and let Safe Path handle this sequence t_ptr = t_save; o_ptr = o_save; e_ptr = e_save; break; } { const uint8_t* src_lit = l_ptr; uint8_t* dst_lit = d_ptr; zxc_copy32(dst_lit, src_lit); if (UNLIKELY(ll > ZXC_PAD_SIZE)) { dst_lit += ZXC_PAD_SIZE; src_lit += ZXC_PAD_SIZE; size_t rem = ll - ZXC_PAD_SIZE; while (rem > ZXC_PAD_SIZE) { zxc_copy32(dst_lit, src_lit); dst_lit += ZXC_PAD_SIZE; src_lit += ZXC_PAD_SIZE; rem -= ZXC_PAD_SIZE; } zxc_copy32(dst_lit, src_lit); } l_ptr += ll; d_ptr += ll; written += ll; } { // Skip check if written >= bounds_threshold (256 for 8-bit, 65536 for 16-bit) if (UNLIKELY(written < bounds_threshold && offset > written)) return ZXC_ERROR_BAD_OFFSET; const uint8_t* match_src = d_ptr - offset; if (LIKELY(offset >= ZXC_PAD_SIZE)) { zxc_copy32(d_ptr, match_src); if (UNLIKELY(ml > ZXC_PAD_SIZE)) { uint8_t* out = d_ptr + ZXC_PAD_SIZE; const uint8_t* ref = match_src + ZXC_PAD_SIZE; size_t rem = ml - ZXC_PAD_SIZE; while (rem > ZXC_PAD_SIZE) { zxc_copy32(out, ref); out += ZXC_PAD_SIZE; ref += ZXC_PAD_SIZE; rem -= ZXC_PAD_SIZE; } zxc_copy32(out, ref); } d_ptr += ml; written += ml; } else if (offset == 1) { ZXC_MEMSET(d_ptr, match_src[0], ml); d_ptr += ml; written += ml; } else { for (size_t i = 0; i < ml; i++) d_ptr[i] = match_src[i]; d_ptr += ml; written += ml; } } n_seq--; } // --- Safe Path for Remaining Sequences --- while (n_seq > 0) { uint8_t token = *t_ptr++; uint32_t ll = token >> ZXC_TOKEN_LIT_BITS; uint32_t ml = token & ZXC_TOKEN_ML_MASK; uint32_t offset = ZXC_LZ_OFFSET_BIAS; if (gh.enc_off == 1) { offset += *o_ptr++; // 1-byte offset (biased) } else { offset += zxc_le16(o_ptr); // 2-byte offset (biased) o_ptr += 2; } if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) ll += zxc_read_varint(&e_ptr, e_end); if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) ml += zxc_read_varint(&e_ptr, e_end); ml += ZXC_LZ_MIN_MATCH_LEN; if (UNLIKELY(d_ptr + ll > d_end || l_ptr + ll > l_end)) return ZXC_ERROR_OVERFLOW; ZXC_MEMCPY(d_ptr, l_ptr, ll); l_ptr += ll; d_ptr += ll; const uint8_t* match_src = d_ptr - offset; if (UNLIKELY(match_src < dst || d_ptr + ml > d_end)) return ZXC_ERROR_BAD_OFFSET; if (offset < ml) { for (size_t i = 0; i < ml; i++) d_ptr[i] = match_src[i]; } else { ZXC_MEMCPY(d_ptr, match_src, ml); } d_ptr += ml; n_seq--; } // --- Trailing Literals --- // Copy remaining literals from source stream (literal exhaustion) if (UNLIKELY(l_ptr > l_end)) return ZXC_ERROR_CORRUPT_DATA; const size_t remaining_literals = (size_t)(l_end - l_ptr); if (remaining_literals > 0) { if (UNLIKELY(d_ptr + remaining_literals > d_end)) return ZXC_ERROR_OVERFLOW; ZXC_MEMCPY(d_ptr, l_ptr, remaining_literals); d_ptr += remaining_literals; } return (int)(d_ptr - dst); } /** * @brief Decodes a GHI (General High) format compressed block. * * This function handles the decoding of a compressed block formatted with the * internal GHI structure. The decompressed size is derived from Section * Descriptors within the compressed payload. * * @param[in] ctx Pointer to the decompression context (unused in current implementation). * @param[in] src Pointer to the source buffer containing compressed data. * @param[in] src_size Size of the source buffer in bytes. * @param[out] dst Pointer to the destination buffer for decompressed data. * @param[in] dst_capacity Capacity of the destination buffer in bytes. * @return int Returns the number of bytes written on success, or a negative zxc_error_t code on * failure. */ /** * @brief Unified GHI decoder body shared by the fast and safe variants. * * @p safe must be a compile-time constant (0 or 1). The two 4x-unrolled loops * are duplicated verbatim inside @c if(safe)/else branches so that each * variant keeps its own single-assignment @c const save pointers. */ static ZXC_ALWAYS_INLINE int zxc_decode_block_ghi_impl(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_size, uint8_t* RESTRICT dst, const size_t dst_capacity, const int safe) { (void)ctx; zxc_gnr_header_t gh; zxc_section_desc_t desc[ZXC_GHI_SECTIONS]; if (UNLIKELY(zxc_read_ghi_header_and_desc(src, src_size, &gh, desc) != ZXC_OK)) return ZXC_ERROR_BAD_HEADER; const uint8_t* p_curr = src + ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; // --- Stream Pointers & Validation --- const size_t sz_lit = (uint32_t)desc[0].sizes; const size_t sz_seqs = (uint32_t)desc[1].sizes; const size_t sz_exts = (uint32_t)desc[2].sizes; const uint8_t* l_ptr = p_curr; const uint8_t* l_end = l_ptr + sz_lit; p_curr += sz_lit; const uint8_t* seq_ptr = p_curr; const uint8_t* extras_ptr = p_curr + sz_seqs; const uint8_t* const extras_end = extras_ptr + sz_exts; // Validate streams don't overflow source buffer + // Validate sequence stream size matches sequence count if (UNLIKELY((extras_end != src + src_size) || ((uint64_t)sz_seqs < (uint64_t)gh.n_sequences * 4))) return ZXC_ERROR_CORRUPT_DATA; uint8_t* d_ptr = dst; const uint8_t* const d_end = dst + dst_capacity; const uint8_t* const d_end_safe = d_end - (ZXC_PAD_SIZE * 4); // 128 // Safety margin for 4x unrolled loop: 4 * (ZXC_SEQ_LL_MASK LL + // ZXC_SEQ_ML_MASK+ZXC_LZ_MIN_MATCH_LEN ML) + ZXC_PAD_SIZE Pad = 4 x (255 + 255 + 5) + 32 = 2092 const uint8_t* const d_end_fast = d_end - ZXC_DECOMPRESS_TAIL_PAD; // 2112 // Literal stream safe thresholds for GHI loops. // Without varint extension, max ll per sequence = ZXC_SEQ_LL_MASK - 1 = 254. // For 4 sequences: 4 * 254 = 1016. With this margin, l_ptr checks are only needed // on the cold varint path, keeping the hot path free of l_ptr overhead. const size_t ghi_margin_4x = 4 * (ZXC_SEQ_LL_MASK - 1); // 1016 const size_t ghi_margin_1x = ZXC_SEQ_LL_MASK - 1; // 254 const uint8_t* const l_end_safe_4x = (sz_lit > ghi_margin_4x) ? l_end - ghi_margin_4x : l_ptr; const uint8_t* const l_end_safe_1x = (sz_lit > ghi_margin_1x) ? l_end - ghi_margin_1x : l_ptr; uint32_t n_seq = gh.n_sequences; // Track bytes written for offset validation // For 1-byte offsets (enc_off==1): validate until 256 bytes written (max 8-bit offset) // For 2-byte offsets (enc_off==0): validate until 65536 bytes written (max 16-bit offset) // After threshold, all offsets are guaranteed valid (can't exceed written bytes) size_t written = 0; // --- SAFE Loop: offset validation until threshold (4x unroll) --- // Since offset is 16-bit, threshold is 65536. // For 1-byte offsets (enc_off==1): validate until 256 bytes written // For 2-byte offsets (enc_off==0): validate until 65536 bytes written const size_t bounds_threshold = (gh.enc_off == 1) ? (1U << 8) : (1U << 16); if (safe) { /* SAFE variant: save per-batch state so an OVERFLOW can rollback and * hand over to the 1x loop / Safe Path. Wild writes already committed * are deterministically overwritten when the 1x loop replays. */ while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x && written < bounds_threshold) { const uint8_t* const t_save = seq_ptr; const uint8_t* const e_save = extras_ptr; uint8_t* const d_save = d_ptr; const uint8_t* const l_save = l_ptr; const size_t w_save = written; uint32_t s1 = zxc_le32(seq_ptr); uint32_t s2 = zxc_le32(seq_ptr + 4); uint32_t s3 = zxc_le32(seq_ptr + 8); uint32_t s4 = zxc_le32(seq_ptr + 12); seq_ptr += 16; uint32_t ll1 = (uint32_t)(s1 >> 24); if (UNLIKELY(ll1 == ZXC_SEQ_LL_MASK)) { ll1 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(l_ptr + ll1 > l_end || d_ptr + ll1 + ZXC_PAD_SIZE > d_end)) goto rollback_safe_4x; } uint32_t m1b = (uint32_t)((s1 >> 16) & 0xFF); uint32_t ml1 = m1b + ZXC_LZ_MIN_MATCH_LEN; if (UNLIKELY(m1b == ZXC_SEQ_ML_MASK)) { ml1 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(d_ptr + ll1 + ml1 + ZXC_PAD_SIZE > d_end)) goto rollback_safe_4x; } uint32_t off1 = (uint32_t)(s1 & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; DECODE_SEQ_SAFE(ll1, ml1, off1); uint32_t ll2 = (uint32_t)(s2 >> 24); if (UNLIKELY(ll2 == ZXC_SEQ_LL_MASK)) { ll2 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(l_ptr + ll2 > l_end || d_ptr + ll2 + ZXC_PAD_SIZE > d_end)) goto rollback_safe_4x; } uint32_t m2b = (uint32_t)((s2 >> 16) & 0xFF); uint32_t ml2 = m2b + ZXC_LZ_MIN_MATCH_LEN; if (UNLIKELY(m2b == ZXC_SEQ_ML_MASK)) { ml2 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(d_ptr + ll2 + ml2 + ZXC_PAD_SIZE > d_end)) goto rollback_safe_4x; } uint32_t off2 = (uint32_t)(s2 & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; DECODE_SEQ_SAFE(ll2, ml2, off2); uint32_t ll3 = (uint32_t)(s3 >> 24); if (UNLIKELY(ll3 == ZXC_SEQ_LL_MASK)) { ll3 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(l_ptr + ll3 > l_end || d_ptr + ll3 + ZXC_PAD_SIZE > d_end)) goto rollback_safe_4x; } uint32_t m3b = (uint32_t)((s3 >> 16) & 0xFF); uint32_t ml3 = m3b + ZXC_LZ_MIN_MATCH_LEN; if (UNLIKELY(m3b == ZXC_SEQ_ML_MASK)) { ml3 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(d_ptr + ll3 + ml3 + ZXC_PAD_SIZE > d_end)) goto rollback_safe_4x; } uint32_t off3 = (uint32_t)(s3 & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; DECODE_SEQ_SAFE(ll3, ml3, off3); uint32_t ll4 = (uint32_t)(s4 >> 24); if (UNLIKELY(ll4 == ZXC_SEQ_LL_MASK)) { ll4 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(l_ptr + ll4 > l_end || d_ptr + ll4 + ZXC_PAD_SIZE > d_end)) goto rollback_safe_4x; } uint32_t m4b = (uint32_t)((s4 >> 16) & 0xFF); uint32_t ml4 = m4b + ZXC_LZ_MIN_MATCH_LEN; if (UNLIKELY(m4b == ZXC_SEQ_ML_MASK)) { ml4 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(d_ptr + ll4 + ml4 + ZXC_PAD_SIZE > d_end)) goto rollback_safe_4x; } uint32_t off4 = (uint32_t)(s4 & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; DECODE_SEQ_SAFE(ll4, ml4, off4); n_seq -= 4; continue; rollback_safe_4x: seq_ptr = t_save; extras_ptr = e_save; d_ptr = d_save; l_ptr = l_save; written = w_save; break; } } else { while (n_seq >= 4 && d_ptr < d_end_safe && l_ptr < l_end_safe_4x && written < bounds_threshold) { uint32_t s1 = zxc_le32(seq_ptr); uint32_t s2 = zxc_le32(seq_ptr + 4); uint32_t s3 = zxc_le32(seq_ptr + 8); uint32_t s4 = zxc_le32(seq_ptr + 12); seq_ptr += 16; uint32_t ll1 = (uint32_t)(s1 >> 24); if (UNLIKELY(ll1 == ZXC_SEQ_LL_MASK)) { ll1 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(l_ptr + ll1 > l_end || d_ptr + ll1 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } uint32_t m1b = (uint32_t)((s1 >> 16) & 0xFF); uint32_t ml1 = m1b + ZXC_LZ_MIN_MATCH_LEN; if (UNLIKELY(m1b == ZXC_SEQ_ML_MASK)) { ml1 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(d_ptr + ll1 + ml1 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } uint32_t off1 = (uint32_t)(s1 & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; DECODE_SEQ_SAFE(ll1, ml1, off1); uint32_t ll2 = (uint32_t)(s2 >> 24); if (UNLIKELY(ll2 == ZXC_SEQ_LL_MASK)) { ll2 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(l_ptr + ll2 > l_end || d_ptr + ll2 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } uint32_t m2b = (uint32_t)((s2 >> 16) & 0xFF); uint32_t ml2 = m2b + ZXC_LZ_MIN_MATCH_LEN; if (UNLIKELY(m2b == ZXC_SEQ_ML_MASK)) { ml2 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(d_ptr + ll2 + ml2 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } uint32_t off2 = (uint32_t)(s2 & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; DECODE_SEQ_SAFE(ll2, ml2, off2); uint32_t ll3 = (uint32_t)(s3 >> 24); if (UNLIKELY(ll3 == ZXC_SEQ_LL_MASK)) { ll3 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(l_ptr + ll3 > l_end || d_ptr + ll3 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } uint32_t m3b = (uint32_t)((s3 >> 16) & 0xFF); uint32_t ml3 = m3b + ZXC_LZ_MIN_MATCH_LEN; if (UNLIKELY(m3b == ZXC_SEQ_ML_MASK)) { ml3 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(d_ptr + ll3 + ml3 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } uint32_t off3 = (uint32_t)(s3 & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; DECODE_SEQ_SAFE(ll3, ml3, off3); uint32_t ll4 = (uint32_t)(s4 >> 24); if (UNLIKELY(ll4 == ZXC_SEQ_LL_MASK)) { ll4 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(l_ptr + ll4 > l_end || d_ptr + ll4 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } uint32_t m4b = (uint32_t)((s4 >> 16) & 0xFF); uint32_t ml4 = m4b + ZXC_LZ_MIN_MATCH_LEN; if (UNLIKELY(m4b == ZXC_SEQ_ML_MASK)) { ml4 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(d_ptr + ll4 + ml4 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } uint32_t off4 = (uint32_t)(s4 & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; DECODE_SEQ_SAFE(ll4, ml4, off4); n_seq -= 4; } } // --- SAFE Loop tail: remaining sequences with offset validation (1x) --- while (n_seq > 0 && d_ptr < d_end_safe && written < bounds_threshold) { uint32_t seq = zxc_le32(seq_ptr); seq_ptr += 4; uint32_t ll = (uint32_t)(seq >> 24); if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) ll += zxc_read_varint(&extras_ptr, extras_end); uint32_t m_bits = (uint32_t)((seq >> 16) & 0xFF); uint32_t ml = m_bits + ZXC_LZ_MIN_MATCH_LEN; if (UNLIKELY(m_bits == ZXC_SEQ_ML_MASK)) ml += zxc_read_varint(&extras_ptr, extras_end); uint32_t offset = (uint32_t)(seq & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; // Strict bounds check: sequence must fit, AND wild copies must not overshoot // Check both destination (d_ptr) and source literal stream (l_ptr) if (UNLIKELY(d_ptr + ll + ml + ZXC_PAD_SIZE > d_end || l_ptr + ll + ZXC_PAD_SIZE > l_end)) { // Fallback to exact copy (slow but safe) if (UNLIKELY(d_ptr + ll > d_end || l_ptr + ll > l_end)) return ZXC_ERROR_OVERFLOW; ZXC_MEMCPY(d_ptr, l_ptr, ll); l_ptr += ll; d_ptr += ll; written += ll; if (UNLIKELY(offset > written || d_ptr + ml > d_end)) return ZXC_ERROR_BAD_OFFSET; const uint8_t* match_src = d_ptr - offset; if (offset < ml) { for (size_t i = 0; i < ml; i++) d_ptr[i] = match_src[i]; } else { ZXC_MEMCPY(d_ptr, match_src, ml); } d_ptr += ml; written += ml; } else { // Safe to process with wild copies DECODE_SEQ_SAFE(ll, ml, offset); } n_seq--; } // --- FAST Loop: After threshold, check large margin to avoid individual bounds checks --- if (safe) { while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x) { const uint8_t* const t_save = seq_ptr; const uint8_t* const e_save = extras_ptr; uint8_t* const d_save = d_ptr; const uint8_t* const l_save = l_ptr; uint32_t s1 = zxc_le32(seq_ptr); uint32_t s2 = zxc_le32(seq_ptr + 4); uint32_t s3 = zxc_le32(seq_ptr + 8); uint32_t s4 = zxc_le32(seq_ptr + 12); seq_ptr += 16; // Prefetch ahead in literal and extras streams to hide memory latency ZXC_PREFETCH_READ(l_ptr + ZXC_CACHE_LINE_SIZE); uint32_t ll1 = (uint32_t)(s1 >> 24); if (UNLIKELY(ll1 == ZXC_SEQ_LL_MASK)) { ll1 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(l_ptr + ll1 > l_end || d_ptr + ll1 + ZXC_PAD_SIZE > d_end)) goto rollback_fast_4x; } uint32_t m1b = (uint32_t)((s1 >> 16) & 0xFF); uint32_t ml1 = m1b + ZXC_LZ_MIN_MATCH_LEN; if (UNLIKELY(m1b == ZXC_SEQ_ML_MASK)) { ml1 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(d_ptr + ll1 + ml1 + ZXC_PAD_SIZE > d_end)) goto rollback_fast_4x; } uint32_t off1 = (uint32_t)(s1 & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; DECODE_SEQ_FAST(ll1, ml1, off1); uint32_t ll2 = (uint32_t)(s2 >> 24); if (UNLIKELY(ll2 == ZXC_SEQ_LL_MASK)) { ll2 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(l_ptr + ll2 > l_end || d_ptr + ll2 + ZXC_PAD_SIZE > d_end)) goto rollback_fast_4x; } uint32_t m2b = (uint32_t)((s2 >> 16) & 0xFF); uint32_t ml2 = m2b + ZXC_LZ_MIN_MATCH_LEN; if (UNLIKELY(m2b == ZXC_SEQ_ML_MASK)) { ml2 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(d_ptr + ll2 + ml2 + ZXC_PAD_SIZE > d_end)) goto rollback_fast_4x; } uint32_t off2 = (uint32_t)(s2 & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; DECODE_SEQ_FAST(ll2, ml2, off2); uint32_t ll3 = (uint32_t)(s3 >> 24); if (UNLIKELY(ll3 == ZXC_SEQ_LL_MASK)) { ll3 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(l_ptr + ll3 > l_end || d_ptr + ll3 + ZXC_PAD_SIZE > d_end)) goto rollback_fast_4x; } uint32_t m3b = (uint32_t)((s3 >> 16) & 0xFF); uint32_t ml3 = m3b + ZXC_LZ_MIN_MATCH_LEN; if (UNLIKELY(m3b == ZXC_SEQ_ML_MASK)) { ml3 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(d_ptr + ll3 + ml3 + ZXC_PAD_SIZE > d_end)) goto rollback_fast_4x; } uint32_t off3 = (uint32_t)(s3 & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; DECODE_SEQ_FAST(ll3, ml3, off3); uint32_t ll4 = (uint32_t)(s4 >> 24); if (UNLIKELY(ll4 == ZXC_SEQ_LL_MASK)) { ll4 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(l_ptr + ll4 > l_end || d_ptr + ll4 + ZXC_PAD_SIZE > d_end)) goto rollback_fast_4x; } uint32_t m4b = (uint32_t)((s4 >> 16) & 0xFF); uint32_t ml4 = m4b + ZXC_LZ_MIN_MATCH_LEN; if (UNLIKELY(m4b == ZXC_SEQ_ML_MASK)) { ml4 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(d_ptr + ll4 + ml4 + ZXC_PAD_SIZE > d_end)) goto rollback_fast_4x; } uint32_t off4 = (uint32_t)(s4 & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; DECODE_SEQ_FAST(ll4, ml4, off4); n_seq -= 4; continue; rollback_fast_4x: seq_ptr = t_save; extras_ptr = e_save; d_ptr = d_save; l_ptr = l_save; break; } } else { while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x) { uint32_t s1 = zxc_le32(seq_ptr); uint32_t s2 = zxc_le32(seq_ptr + 4); uint32_t s3 = zxc_le32(seq_ptr + 8); uint32_t s4 = zxc_le32(seq_ptr + 12); seq_ptr += 16; // Prefetch ahead in literal and extras streams to hide memory latency ZXC_PREFETCH_READ(l_ptr + ZXC_CACHE_LINE_SIZE); uint32_t ll1 = (uint32_t)(s1 >> 24); if (UNLIKELY(ll1 == ZXC_SEQ_LL_MASK)) { ll1 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(l_ptr + ll1 > l_end || d_ptr + ll1 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } uint32_t m1b = (uint32_t)((s1 >> 16) & 0xFF); uint32_t ml1 = m1b + ZXC_LZ_MIN_MATCH_LEN; if (UNLIKELY(m1b == ZXC_SEQ_ML_MASK)) { ml1 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(d_ptr + ll1 + ml1 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } uint32_t off1 = (uint32_t)(s1 & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; DECODE_SEQ_FAST(ll1, ml1, off1); uint32_t ll2 = (uint32_t)(s2 >> 24); if (UNLIKELY(ll2 == ZXC_SEQ_LL_MASK)) { ll2 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(l_ptr + ll2 > l_end || d_ptr + ll2 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } uint32_t m2b = (uint32_t)((s2 >> 16) & 0xFF); uint32_t ml2 = m2b + ZXC_LZ_MIN_MATCH_LEN; if (UNLIKELY(m2b == ZXC_SEQ_ML_MASK)) { ml2 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(d_ptr + ll2 + ml2 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } uint32_t off2 = (uint32_t)(s2 & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; DECODE_SEQ_FAST(ll2, ml2, off2); uint32_t ll3 = (uint32_t)(s3 >> 24); if (UNLIKELY(ll3 == ZXC_SEQ_LL_MASK)) { ll3 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(l_ptr + ll3 > l_end || d_ptr + ll3 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } uint32_t m3b = (uint32_t)((s3 >> 16) & 0xFF); uint32_t ml3 = m3b + ZXC_LZ_MIN_MATCH_LEN; if (UNLIKELY(m3b == ZXC_SEQ_ML_MASK)) { ml3 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(d_ptr + ll3 + ml3 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } uint32_t off3 = (uint32_t)(s3 & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; DECODE_SEQ_FAST(ll3, ml3, off3); uint32_t ll4 = (uint32_t)(s4 >> 24); if (UNLIKELY(ll4 == ZXC_SEQ_LL_MASK)) { ll4 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(l_ptr + ll4 > l_end || d_ptr + ll4 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } uint32_t m4b = (uint32_t)((s4 >> 16) & 0xFF); uint32_t ml4 = m4b + ZXC_LZ_MIN_MATCH_LEN; if (UNLIKELY(m4b == ZXC_SEQ_ML_MASK)) { ml4 += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(d_ptr + ll4 + ml4 + ZXC_PAD_SIZE > d_end)) return ZXC_ERROR_OVERFLOW; } uint32_t off4 = (uint32_t)(s4 & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; DECODE_SEQ_FAST(ll4, ml4, off4); n_seq -= 4; } } // --- Remaining 1 sequence (Fast Path) --- while (n_seq > 0 && d_ptr < d_end_safe && l_ptr < l_end_safe_1x) { // Save state for fallback const uint8_t* seq_save = seq_ptr; const uint8_t* ext_save = extras_ptr; const uint32_t seq = zxc_le32(seq_ptr); seq_ptr += 4; uint32_t ll = (uint32_t)(seq >> 24); if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { ll += zxc_read_varint(&extras_ptr, extras_end); if (UNLIKELY(l_ptr + ll > l_end)) { seq_ptr = seq_save; extras_ptr = ext_save; break; } } uint32_t m_bits = (uint32_t)((seq >> 16) & 0xFF); uint32_t ml = m_bits + ZXC_LZ_MIN_MATCH_LEN; if (UNLIKELY(m_bits == ZXC_SEQ_ML_MASK)) ml += zxc_read_varint(&extras_ptr, extras_end); // Strict bounds checks (including wild copy overrun safety) if (UNLIKELY(d_ptr + ll + ml + ZXC_PAD_SIZE > d_end)) { // Restore state and break to Safe Path seq_ptr = seq_save; extras_ptr = ext_save; break; } uint32_t offset = (uint32_t)(seq & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; { const uint8_t* src_lit = l_ptr; uint8_t* dst_lit = d_ptr; zxc_copy32(dst_lit, src_lit); if (UNLIKELY(ll > ZXC_PAD_SIZE)) { dst_lit += ZXC_PAD_SIZE; src_lit += ZXC_PAD_SIZE; size_t rem = ll - ZXC_PAD_SIZE; while (rem > ZXC_PAD_SIZE) { zxc_copy32(dst_lit, src_lit); dst_lit += ZXC_PAD_SIZE; src_lit += ZXC_PAD_SIZE; rem -= ZXC_PAD_SIZE; } zxc_copy32(dst_lit, src_lit); } l_ptr += ll; d_ptr += ll; written += ll; } { // Skip check if written >= bounds_threshold (256 for 8-bit, 65536 for 16-bit) if (UNLIKELY(written < bounds_threshold && offset > written)) return ZXC_ERROR_BAD_OFFSET; const uint8_t* match_src = d_ptr - offset; if (LIKELY(offset >= ZXC_PAD_SIZE)) { zxc_copy32(d_ptr, match_src); if (UNLIKELY(ml > ZXC_PAD_SIZE)) { uint8_t* out = d_ptr + ZXC_PAD_SIZE; const uint8_t* ref = match_src + ZXC_PAD_SIZE; size_t rem = ml - ZXC_PAD_SIZE; while (rem > ZXC_PAD_SIZE) { zxc_copy32(out, ref); out += ZXC_PAD_SIZE; ref += ZXC_PAD_SIZE; rem -= ZXC_PAD_SIZE; } zxc_copy32(out, ref); } d_ptr += ml; written += ml; } else if (offset == 1) { ZXC_MEMSET(d_ptr, match_src[0], ml); d_ptr += ml; written += ml; } else { for (size_t i = 0; i < ml; i++) d_ptr[i] = match_src[i]; d_ptr += ml; written += ml; } } n_seq--; } // --- Safe Path for Remaining Sequences --- while (n_seq > 0) { uint32_t seq = zxc_le32(seq_ptr); seq_ptr += 4; uint32_t ll = (uint32_t)(seq >> 24); if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) ll += zxc_read_varint(&extras_ptr, extras_end); uint32_t m_bits = (uint32_t)((seq >> 16) & 0xFF); uint32_t ml = m_bits + ZXC_LZ_MIN_MATCH_LEN; if (UNLIKELY(m_bits == ZXC_SEQ_ML_MASK)) ml += zxc_read_varint(&extras_ptr, extras_end); uint32_t offset = (uint32_t)(seq & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; if (UNLIKELY(d_ptr + ll > d_end || l_ptr + ll > l_end)) return ZXC_ERROR_OVERFLOW; ZXC_MEMCPY(d_ptr, l_ptr, ll); l_ptr += ll; d_ptr += ll; const uint8_t* match_src = d_ptr - offset; if (UNLIKELY(match_src < dst || d_ptr + ml > d_end)) return ZXC_ERROR_BAD_OFFSET; if (offset < ml) { for (size_t i = 0; i < ml; i++) d_ptr[i] = match_src[i]; } else { ZXC_MEMCPY(d_ptr, match_src, ml); } d_ptr += ml; n_seq--; } // --- Trailing Literals --- // Copy remaining literals from source stream (literal exhaustion) if (UNLIKELY(l_ptr > l_end)) return ZXC_ERROR_CORRUPT_DATA; const size_t remaining_literals = (size_t)(l_end - l_ptr); if (remaining_literals > 0) { if (UNLIKELY(d_ptr + remaining_literals > d_end)) return ZXC_ERROR_OVERFLOW; ZXC_MEMCPY(d_ptr, l_ptr, remaining_literals); d_ptr += remaining_literals; } return (int)(d_ptr - dst); } static int zxc_decode_block_ghi(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_size, uint8_t* RESTRICT dst, const size_t dst_capacity) { return zxc_decode_block_ghi_impl(ctx, src, src_size, dst, dst_capacity, 0); } static int zxc_decode_block_glo(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_size, uint8_t* RESTRICT dst, const size_t dst_capacity) { return zxc_decode_block_glo_impl(ctx, src, src_size, dst, dst_capacity, 0); } static int zxc_decode_block_glo_safe(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_size, uint8_t* RESTRICT dst, const size_t dst_capacity) { return zxc_decode_block_glo_impl(ctx, src, src_size, dst, dst_capacity, 1); } static int zxc_decode_block_ghi_safe(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_size, uint8_t* RESTRICT dst, const size_t dst_capacity) { return zxc_decode_block_ghi_impl(ctx, src, src_size, dst, dst_capacity, 1); } #undef DECODE_SEQ_FAST #undef DECODE_SEQ_SAFE #undef DECODE_COPY_MATCH #undef DECODE_COPY_LITERALS // cppcheck-suppress unusedFunction int zxc_decompress_chunk_wrapper(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { if (UNLIKELY(src_sz < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; const uint8_t type = src[0]; const uint32_t comp_sz = zxc_le32(src + 3); const int has_crc = ctx->checksum_enabled; // Check bounds: Header + Body + Checksum(if any) const size_t expected_sz = (size_t)ZXC_BLOCK_HEADER_SIZE + comp_sz + (has_crc ? ZXC_BLOCK_CHECKSUM_SIZE : 0); if (UNLIKELY(src_sz < expected_sz)) return ZXC_ERROR_SRC_TOO_SMALL; const uint8_t* data = src + ZXC_BLOCK_HEADER_SIZE; if (has_crc) { const uint32_t stored = zxc_le32(data + comp_sz); const uint32_t calc = zxc_checksum(data, comp_sz, ZXC_CHECKSUM_RAPIDHASH); if (UNLIKELY(stored != calc)) return ZXC_ERROR_BAD_CHECKSUM; } int decoded_sz = ZXC_ERROR_BAD_BLOCK_TYPE; switch (type) { case ZXC_BLOCK_GLO: decoded_sz = zxc_decode_block_glo(ctx, data, comp_sz, dst, dst_cap); break; case ZXC_BLOCK_GHI: decoded_sz = zxc_decode_block_ghi(ctx, data, comp_sz, dst, dst_cap); break; case ZXC_BLOCK_RAW: // For RAW blocks, comp_sz == raw_sz (uncompressed data stored as-is) if (UNLIKELY(comp_sz > dst_cap)) return ZXC_ERROR_DST_TOO_SMALL; ZXC_MEMCPY(dst, data, comp_sz); decoded_sz = (int)comp_sz; break; case ZXC_BLOCK_NUM: decoded_sz = zxc_decode_block_num(data, comp_sz, dst, dst_cap); break; case ZXC_BLOCK_EOF: // EOF should be handled by the dispatcher, not here return ZXC_ERROR_CORRUPT_DATA; default: return ZXC_ERROR_BAD_BLOCK_TYPE; } return decoded_sz; } // cppcheck-suppress unusedFunction int zxc_decompress_chunk_wrapper_safe(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { if (UNLIKELY(src_sz < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; const uint8_t type = src[0]; const uint32_t comp_sz = zxc_le32(src + 3); const int has_crc = ctx->checksum_enabled; const size_t expected_sz = (size_t)ZXC_BLOCK_HEADER_SIZE + comp_sz + (has_crc ? ZXC_BLOCK_CHECKSUM_SIZE : 0); if (UNLIKELY(src_sz < expected_sz)) return ZXC_ERROR_SRC_TOO_SMALL; const uint8_t* data = src + ZXC_BLOCK_HEADER_SIZE; if (has_crc) { const uint32_t stored = zxc_le32(data + comp_sz); const uint32_t calc = zxc_checksum(data, comp_sz, ZXC_CHECKSUM_RAPIDHASH); if (UNLIKELY(stored != calc)) return ZXC_ERROR_BAD_CHECKSUM; } switch (type) { case ZXC_BLOCK_GLO: return zxc_decode_block_glo_safe(ctx, data, comp_sz, dst, dst_cap); case ZXC_BLOCK_GHI: return zxc_decode_block_ghi_safe(ctx, data, comp_sz, dst, dst_cap); case ZXC_BLOCK_RAW: if (UNLIKELY(comp_sz > dst_cap)) return ZXC_ERROR_DST_TOO_SMALL; ZXC_MEMCPY(dst, data, comp_sz); return (int)comp_sz; case ZXC_BLOCK_NUM: return zxc_decode_block_num(data, comp_sz, dst, dst_cap); case ZXC_BLOCK_EOF: return ZXC_ERROR_CORRUPT_DATA; default: return ZXC_ERROR_BAD_BLOCK_TYPE; } } zxc-0.11.0/src/lib/zxc_dispatch.c000066400000000000000000001350131520102567100165630ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /** * @file zxc_dispatch.c * @brief Runtime CPU feature detection and SIMD dispatch layer. * * Detects AVX2/AVX512/NEON at runtime and routes compress/decompress calls * to the best available implementation via lazy-initialised function pointers. * Also contains the public one-shot buffer API (@ref zxc_compress, * @ref zxc_decompress, @ref zxc_get_decompressed_size). */ #include "../../include/zxc_error.h" #include "../../include/zxc_seekable.h" #include "zxc_internal.h" /* * ZXC_DISABLE_SIMD => force ZXC_ONLY_DEFAULT so the dispatcher never selects * an AVX2/AVX512/NEON variant. */ #if defined(ZXC_DISABLE_SIMD) && !defined(ZXC_ONLY_DEFAULT) #define ZXC_ONLY_DEFAULT #endif #if defined(_MSC_VER) #include #endif #if defined(__linux__) && (defined(__arm__) || defined(_M_ARM)) #include #include #endif /* * ============================================================================ * PROTOTYPES FOR MULTI-VERSIONED VARIANTS * ============================================================================ * These are compiled in separate translation units with different flags. */ // Decompression Prototypes int zxc_decompress_chunk_wrapper_default(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap); int zxc_decompress_chunk_wrapper_safe_default(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap); #ifndef ZXC_ONLY_DEFAULT #if defined(__x86_64__) || defined(_M_X64) int zxc_decompress_chunk_wrapper_avx2(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap); int zxc_decompress_chunk_wrapper_avx512(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap); int zxc_decompress_chunk_wrapper_safe_avx2(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap); int zxc_decompress_chunk_wrapper_safe_avx512(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap); #elif defined(__aarch64__) || defined(_M_ARM64) || defined(__arm__) || defined(_M_ARM) int zxc_decompress_chunk_wrapper_neon(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap); int zxc_decompress_chunk_wrapper_safe_neon(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap); #endif #endif // Compression Prototypes int zxc_compress_chunk_wrapper_default(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap); // Huffman Prototypes (variant TUs of zxc_huffman.c). The compressor and // decompressor variants resolve their Huffman calls to the matching suffixed // symbol at compile time (zero dispatch overhead in the hot path); the thin // wrappers below expose the un-suffixed names for tests and external callers. int zxc_huf_build_code_lengths_default(const uint32_t* RESTRICT freq, uint8_t* RESTRICT code_len, void* RESTRICT scratch); int zxc_huf_encode_section_default(const uint8_t* RESTRICT literals, const size_t n_literals, const uint8_t* RESTRICT code_len, uint8_t* RESTRICT dst, const size_t dst_cap); int zxc_huf_decode_section_default(const uint8_t* RESTRICT payload, const size_t payload_size, uint8_t* RESTRICT dst, const size_t n_literals); #if defined(__x86_64__) || defined(_M_X64) int zxc_compress_chunk_wrapper_avx2(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap); int zxc_compress_chunk_wrapper_avx512(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap); #elif defined(__aarch64__) || defined(_M_ARM64) || defined(__arm__) || defined(_M_ARM) int zxc_compress_chunk_wrapper_neon(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap); #endif /* * ============================================================================ * CPU DETECTION LOGIC * ============================================================================ */ /** * @enum zxc_cpu_feature_t * @brief Detected CPU SIMD capability level. */ typedef enum { ZXC_CPU_GENERIC = 0, /**< @brief Scalar-only fallback. */ ZXC_CPU_AVX2 = 1, /**< @brief x86-64 AVX2 available. */ ZXC_CPU_AVX512 = 2, /**< @brief x86-64 AVX-512F+BW available. */ ZXC_CPU_NEON = 3 /**< @brief ARM NEON available. */ } zxc_cpu_feature_t; /** * @brief Probes the running CPU for SIMD support. * * Uses CPUID on x86-64 (MSVC and GCC/Clang paths), `getauxval` on * 32-bit ARM Linux, and compile-time constants on AArch64. * * @return The highest @ref zxc_cpu_feature_t level supported. */ // LCOV_EXCL_START static zxc_cpu_feature_t zxc_detect_cpu_features(void) { #ifdef ZXC_ONLY_DEFAULT return ZXC_CPU_GENERIC; #else zxc_cpu_feature_t features = ZXC_CPU_GENERIC; #if defined(__x86_64__) || defined(_M_X64) #if defined(_MSC_VER) // MSVC detection using __cpuid // Function ID 1: EAX=1. ECX: Bit 28=AVX. // Function ID 7: EAX=7, ECX=0. EBX: Bit 5=AVX2, Bit 16=AVX512F, Bit 30=AVX512BW. int regs[4]; int avx = 0; int avx2 = 0; int avx512 = 0; __cpuid(regs, 1); if (regs[2] & (1 << 28)) avx = 1; if (avx) { __cpuidex(regs, 7, 0); if (regs[1] & (1 << 5)) avx2 = 1; if ((regs[1] & (1 << 16)) && (regs[1] & (1 << 30))) avx512 = 1; } if (avx512) { features = ZXC_CPU_AVX512; } else if (avx2) { features = ZXC_CPU_AVX2; } #else // GCC/Clang built-in detection __builtin_cpu_init(); if (__builtin_cpu_supports("avx512f") && __builtin_cpu_supports("avx512bw")) { features = ZXC_CPU_AVX512; } else if (__builtin_cpu_supports("avx2")) { features = ZXC_CPU_AVX2; } #endif #elif defined(__aarch64__) || defined(_M_ARM64) // ARM64 usually guarantees NEON features = ZXC_CPU_NEON; #elif defined(__arm__) || defined(_M_ARM) // ARM32 Runtime detection for Linux #if defined(__linux__) const unsigned long hwcaps = getauxval(AT_HWCAP); if (hwcaps & HWCAP_NEON) { features = ZXC_CPU_NEON; } #else // Fallback for non-Linux: rely on compiler flags. // If compiled with -mfpu=neon, we assume target supports it. // Otherwise, safe default is GENERIC. #if defined(__ARM_NEON) features = ZXC_CPU_NEON; #endif #endif #endif return features; #endif } // LCOV_EXCL_STOP /* * ============================================================================ * DISPATCHERS * ============================================================================ * We use a function pointer initialized on first use (lazy initialization). */ /** @brief Function pointer type for the chunk decompressor. */ typedef int (*zxc_decompress_func_t)(zxc_cctx_t* RESTRICT, const uint8_t* RESTRICT, const size_t, uint8_t* RESTRICT, const size_t); /** @brief Function pointer type for the chunk compressor. */ typedef int (*zxc_compress_func_t)(zxc_cctx_t* RESTRICT, const uint8_t* RESTRICT, const size_t, uint8_t* RESTRICT, const size_t); /** @brief Lazily-resolved pointer to the best decompression variant. */ static ZXC_ATOMIC zxc_decompress_func_t zxc_decompress_ptr = (zxc_decompress_func_t)0; /** @brief Lazily-resolved pointer to the best safe-decompression variant. */ static ZXC_ATOMIC zxc_decompress_func_t zxc_decompress_safe_ptr = (zxc_decompress_func_t)0; /** @brief Lazily-resolved pointer to the best compression variant. */ static ZXC_ATOMIC zxc_compress_func_t zxc_compress_ptr = (zxc_compress_func_t)0; /** * @brief First-call initialiser for the decompression dispatcher. * * Detects CPU features, selects the best implementation, stores the * pointer atomically, then tail-calls into it. */ // LCOV_EXCL_START static int zxc_decompress_dispatch_init(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { const zxc_cpu_feature_t cpu = zxc_detect_cpu_features(); zxc_decompress_func_t zxc_decompress_ptr_local = NULL; #ifndef ZXC_ONLY_DEFAULT #if defined(__x86_64__) || defined(_M_X64) if (cpu == ZXC_CPU_AVX512) zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_avx512; else if (cpu == ZXC_CPU_AVX2) zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_avx2; else zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_default; #elif defined(__aarch64__) || defined(_M_ARM64) || defined(__arm__) || defined(_M_ARM) // cppcheck-suppress knownConditionTrueFalse if (cpu == ZXC_CPU_NEON) zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_neon; else zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_default; #else (void)cpu; zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_default; #endif #else (void)cpu; zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_default; #endif #if ZXC_USE_C11_ATOMICS atomic_store_explicit(&zxc_decompress_ptr, zxc_decompress_ptr_local, memory_order_release); #else zxc_decompress_ptr = zxc_decompress_ptr_local; #endif return zxc_decompress_ptr_local(ctx, src, src_sz, dst, dst_cap); } // LCOV_EXCL_STOP /** * @brief First-call initialiser for the safe-decompression dispatcher. * * Mirrors @ref zxc_decompress_dispatch_init but selects the `_safe_*` * decoder variants used by @ref zxc_decompress_block_safe. */ // LCOV_EXCL_START static int zxc_decompress_safe_dispatch_init(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { const zxc_cpu_feature_t cpu = zxc_detect_cpu_features(); zxc_decompress_func_t zxc_decompress_safe_ptr_local = NULL; #ifndef ZXC_ONLY_DEFAULT #if defined(__x86_64__) || defined(_M_X64) if (cpu == ZXC_CPU_AVX512) zxc_decompress_safe_ptr_local = zxc_decompress_chunk_wrapper_safe_avx512; else if (cpu == ZXC_CPU_AVX2) zxc_decompress_safe_ptr_local = zxc_decompress_chunk_wrapper_safe_avx2; else zxc_decompress_safe_ptr_local = zxc_decompress_chunk_wrapper_safe_default; #elif defined(__aarch64__) || defined(_M_ARM64) || defined(__arm__) || defined(_M_ARM) // cppcheck-suppress knownConditionTrueFalse if (cpu == ZXC_CPU_NEON) zxc_decompress_safe_ptr_local = zxc_decompress_chunk_wrapper_safe_neon; else zxc_decompress_safe_ptr_local = zxc_decompress_chunk_wrapper_safe_default; #else (void)cpu; zxc_decompress_safe_ptr_local = zxc_decompress_chunk_wrapper_safe_default; #endif #else (void)cpu; zxc_decompress_safe_ptr_local = zxc_decompress_chunk_wrapper_safe_default; #endif #if ZXC_USE_C11_ATOMICS atomic_store_explicit(&zxc_decompress_safe_ptr, zxc_decompress_safe_ptr_local, memory_order_release); #else zxc_decompress_safe_ptr = zxc_decompress_safe_ptr_local; #endif return zxc_decompress_safe_ptr_local(ctx, src, src_sz, dst, dst_cap); } // LCOV_EXCL_STOP /** * @brief First-call initialiser for the compression dispatcher. * * Detects CPU features, selects the best implementation, stores the * pointer atomically, then tail-calls into it. */ // LCOV_EXCL_START static int zxc_compress_dispatch_init(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { const zxc_cpu_feature_t cpu = zxc_detect_cpu_features(); zxc_compress_func_t zxc_compress_ptr_local = NULL; #ifndef ZXC_ONLY_DEFAULT #if defined(__x86_64__) || defined(_M_X64) if (cpu == ZXC_CPU_AVX512) zxc_compress_ptr_local = zxc_compress_chunk_wrapper_avx512; else if (cpu == ZXC_CPU_AVX2) zxc_compress_ptr_local = zxc_compress_chunk_wrapper_avx2; else zxc_compress_ptr_local = zxc_compress_chunk_wrapper_default; #elif defined(__aarch64__) || defined(_M_ARM64) || defined(__arm__) || defined(_M_ARM) // cppcheck-suppress knownConditionTrueFalse if (cpu == ZXC_CPU_NEON) zxc_compress_ptr_local = zxc_compress_chunk_wrapper_neon; else zxc_compress_ptr_local = zxc_compress_chunk_wrapper_default; #else (void)cpu; zxc_compress_ptr_local = zxc_compress_chunk_wrapper_default; #endif #else (void)cpu; zxc_compress_ptr_local = zxc_compress_chunk_wrapper_default; #endif #if ZXC_USE_C11_ATOMICS atomic_store_explicit(&zxc_compress_ptr, zxc_compress_ptr_local, memory_order_release); #else zxc_compress_ptr = zxc_compress_ptr_local; #endif return zxc_compress_ptr_local(ctx, src, src_sz, dst, dst_cap); } // LCOV_EXCL_STOP /** * @brief Public decompression dispatcher (calls lazily-resolved implementation). * * @param[in,out] ctx Decompression context. * @param[in] src Compressed input chunk (header + payload + optional checksum). * @param[in] src_sz Size of @p src in bytes. * @param[out] dst Destination buffer for decompressed data. * @param[in] dst_cap Capacity of @p dst. * @return Decompressed size in bytes, or a negative @ref zxc_error_t code. */ int zxc_decompress_chunk_wrapper(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { #if ZXC_USE_C11_ATOMICS const zxc_decompress_func_t func = atomic_load_explicit(&zxc_decompress_ptr, memory_order_acquire); #else const zxc_decompress_func_t func = zxc_decompress_ptr; #endif if (UNLIKELY(!func)) return zxc_decompress_dispatch_init(ctx, src, src_sz, dst, dst_cap); return func(ctx, src, src_sz, dst, dst_cap); } /** * @brief Internal safe-decompression dispatcher (strict dst_capacity == uncompressed_size). */ static int zxc_decompress_chunk_wrapper_safe_public(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { #if ZXC_USE_C11_ATOMICS const zxc_decompress_func_t func = atomic_load_explicit(&zxc_decompress_safe_ptr, memory_order_acquire); #else const zxc_decompress_func_t func = zxc_decompress_safe_ptr; #endif if (UNLIKELY(!func)) return zxc_decompress_safe_dispatch_init(ctx, src, src_sz, dst, dst_cap); return func(ctx, src, src_sz, dst, dst_cap); } /** * @brief Public compression dispatcher (calls lazily-resolved implementation). * * @param[in,out] ctx Compression context. * @param[in] src Uncompressed input chunk. * @param[in] src_sz Size of @p src in bytes. * @param[out] dst Destination buffer for compressed data. * @param[in] dst_cap Capacity of @p dst. * @return Compressed size in bytes, or a negative @ref zxc_error_t code. */ int zxc_compress_chunk_wrapper(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { #if ZXC_USE_C11_ATOMICS const zxc_compress_func_t func = atomic_load_explicit(&zxc_compress_ptr, memory_order_acquire); #else const zxc_compress_func_t func = zxc_compress_ptr; #endif if (UNLIKELY(!func)) return zxc_compress_dispatch_init(ctx, src, src_sz, dst, dst_cap); return func(ctx, src, src_sz, dst, dst_cap); } /* * ============================================================================ * HUFFMAN TRAMPOLINES * ============================================================================ * The Huffman codec is built per-variant (default / avx2 / avx512 / neon) * alongside zxc_compress.c and zxc_decompress.c, so the LZ77 stages and the * Huffman stage in a given variant share the same ISA flags (e.g. -mbmi2 on * the AVX2/AVX512 variants). The compress/decompress variant TUs resolve * their Huffman calls to the matching suffixed symbol at compile time, so * the production hot path has zero dispatch overhead. * * These thin wrappers exist only for tests and external callers that link * against the un-suffixed names. They forward to the default (scalar) variant. */ int zxc_huf_build_code_lengths(const uint32_t* RESTRICT freq, uint8_t* RESTRICT code_len, void* RESTRICT scratch) { return zxc_huf_build_code_lengths_default(freq, code_len, scratch); } int zxc_huf_encode_section(const uint8_t* RESTRICT literals, const size_t n_literals, const uint8_t* RESTRICT code_len, uint8_t* RESTRICT dst, const size_t dst_cap) { return zxc_huf_encode_section_default(literals, n_literals, code_len, dst, dst_cap); } int zxc_huf_decode_section(const uint8_t* RESTRICT payload, const size_t payload_size, uint8_t* RESTRICT dst, const size_t n_literals) { return zxc_huf_decode_section_default(payload, payload_size, dst, n_literals); } /* * ============================================================================ * PUBLIC UTILITY API * ============================================================================ * These wrapper functions provide a simplified interface by managing context * allocation and looping over blocks. They call the dispatched wrappers above. */ /** * @brief Compresses an entire buffer in one call. * * Manages context allocation internally, loops over blocks, writes the * file header / EOF block / footer, and accumulates the global checksum. * * @param[in] src Uncompressed input data. * @param[in] src_size Size of @p src in bytes. * @param[out] dst Destination buffer (use zxc_compress_bound() to size). * @param[in] dst_capacity Capacity of @p dst. * @param[in] level Compression level (1-5). * @param[in] checksum_enabled Non-zero to enable per-block and global checksums. * @return Total compressed size in bytes, or a negative @ref zxc_error_t code. */ // cppcheck-suppress unusedFunction int64_t zxc_compress(const void* RESTRICT src, const size_t src_size, void* RESTRICT dst, const size_t dst_capacity, const zxc_compress_opts_t* opts) { if (UNLIKELY(!src || !dst || src_size == 0 || dst_capacity == 0)) return ZXC_ERROR_NULL_INPUT; const int checksum_enabled = opts ? opts->checksum_enabled : 0; const int seekable = opts ? opts->seekable : 0; const int level = (opts && opts->level > 0) ? opts->level : ZXC_LEVEL_DEFAULT; const size_t block_size = (opts && opts->block_size > 0) ? opts->block_size : ZXC_BLOCK_SIZE_DEFAULT; if (UNLIKELY(!zxc_validate_block_size(block_size))) return ZXC_ERROR_BAD_BLOCK_SIZE; const uint8_t* ip = (const uint8_t*)src; uint8_t* op = (uint8_t*)dst; const uint8_t* op_start = op; const uint8_t* op_end = op + dst_capacity; uint32_t global_hash = 0; zxc_cctx_t ctx; // LCOV_EXCL_START if (UNLIKELY(zxc_cctx_init(&ctx, block_size, 1, level, checksum_enabled) != ZXC_OK)) return ZXC_ERROR_MEMORY; // LCOV_EXCL_STOP const int h_val = zxc_write_file_header(op, (size_t)(op_end - op), block_size, checksum_enabled); // LCOV_EXCL_START if (UNLIKELY(h_val < 0)) { zxc_cctx_free(&ctx); return h_val; } // LCOV_EXCL_STOP op += h_val; /* Seekable: dynamic array for per-block compressed sizes */ uint32_t* seek_comp = NULL; uint32_t seek_count = 0; uint32_t seek_cap = 0; if (seekable) { const size_t block_count = src_size / block_size; if (UNLIKELY(block_count > (size_t)UINT32_MAX - 2)) { zxc_cctx_free(&ctx); return ZXC_ERROR_BAD_BLOCK_SIZE; } seek_cap = (uint32_t)(block_count + 2); seek_comp = (uint32_t*)malloc(seek_cap * sizeof(uint32_t)); // LCOV_EXCL_START if (UNLIKELY(!seek_comp)) { zxc_cctx_free(&ctx); return ZXC_ERROR_MEMORY; } // LCOV_EXCL_STOP } size_t pos = 0; while (pos < src_size) { const size_t chunk_len = (src_size - pos > block_size) ? block_size : (src_size - pos); const size_t rem_cap = (size_t)(op_end - op); const int res = zxc_compress_chunk_wrapper(&ctx, ip + pos, chunk_len, op, rem_cap); if (UNLIKELY(res < 0)) { free(seek_comp); zxc_cctx_free(&ctx); return res; } if (checksum_enabled) { // Update Global Hash (Rotation + XOR) // Block checksum is at the end of the written block data if (LIKELY(res >= ZXC_GLOBAL_CHECKSUM_SIZE)) { const uint32_t block_hash = zxc_le32(op + res - ZXC_GLOBAL_CHECKSUM_SIZE); global_hash = zxc_hash_combine_rotate(global_hash, block_hash); } } /* Seekable: record compressed block size */ if (seekable) { // LCOV_EXCL_START if (UNLIKELY(seek_count >= seek_cap)) { seek_cap = seek_cap * 2; uint32_t* nc = (uint32_t*)realloc(seek_comp, seek_cap * sizeof(uint32_t)); if (UNLIKELY(!nc)) { free(seek_comp); zxc_cctx_free(&ctx); return ZXC_ERROR_MEMORY; } seek_comp = nc; } // LCOV_EXCL_STOP seek_comp[seek_count] = (uint32_t)res; seek_count++; } op += res; pos += chunk_len; } zxc_cctx_free(&ctx); // Write EOF Block const size_t rem_cap = (size_t)(op_end - op); const zxc_block_header_t eof_bh = { .block_type = ZXC_BLOCK_EOF, .block_flags = 0, .reserved = 0, .comp_size = 0}; const int eof_val = zxc_write_block_header(op, rem_cap, &eof_bh); // LCOV_EXCL_START if (UNLIKELY(eof_val < 0)) { free(seek_comp); return eof_val; } // LCOV_EXCL_STOP op += eof_val; /* Seekable: write seek table between EOF block and footer */ if (seekable && seek_count > 0) { const size_t st_cap = (size_t)(op_end - op); const int64_t st_val = zxc_write_seek_table(op, st_cap, seek_comp, seek_count); free(seek_comp); if (UNLIKELY(st_val < 0)) return (int64_t)st_val; // LCOV_EXCL_LINE op += st_val; } else { free(seek_comp); } if (UNLIKELY((size_t)(op_end - op) < ZXC_FILE_FOOTER_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; // LCOV_EXCL_LINE // Write 12-byte Footer: [Source Size (8)] + [Global Hash (4)] const int footer_val = zxc_write_file_footer(op, (size_t)(op_end - op), src_size, global_hash, checksum_enabled); if (UNLIKELY(footer_val < 0)) return footer_val; // LCOV_EXCL_LINE op += footer_val; return (int64_t)(op - op_start); } /** * @brief Decompresses an entire buffer in one call. * * Validates the file header and footer, loops over compressed blocks, * and verifies the global checksum when enabled. * * @param[in] src Compressed input data. * @param[in] src_size Size of @p src in bytes. * @param[out] dst Destination buffer for decompressed data. * @param[in] dst_capacity Capacity of @p dst. * @param[in] checksum_enabled Non-zero to verify per-block and global checksums. * @return Total decompressed size in bytes, or a negative @ref zxc_error_t code. */ // cppcheck-suppress unusedFunction int64_t zxc_decompress(const void* RESTRICT src, const size_t src_size, void* RESTRICT dst, const size_t dst_capacity, const zxc_decompress_opts_t* opts) { if (UNLIKELY(!src || !dst || src_size < ZXC_FILE_HEADER_SIZE)) return ZXC_ERROR_NULL_INPUT; const int checksum_enabled = opts ? opts->checksum_enabled : 0; const uint8_t* ip = (const uint8_t*)src; const uint8_t* ip_end = ip + src_size; uint8_t* op = (uint8_t*)dst; const uint8_t* op_start = op; const uint8_t* op_end = op + dst_capacity; size_t runtime_chunk_size = 0; zxc_cctx_t ctx; int file_has_checksums = 0; // File header verification and context initialization if (UNLIKELY(zxc_read_file_header(ip, src_size, &runtime_chunk_size, &file_has_checksums) != ZXC_OK || zxc_cctx_init(&ctx, runtime_chunk_size, 0, 0, file_has_checksums && checksum_enabled) != ZXC_OK)) { return ZXC_ERROR_BAD_HEADER; } ip += ZXC_FILE_HEADER_SIZE; // GLO/GHI wild copies (zxc_copy32) overshoot by up to ZXC_PAD_SIZE bytes. // Decode into a padded scratch buffer, then memcpy the exact result out. const size_t work_sz = runtime_chunk_size + ZXC_PAD_SIZE; if (ctx.work_buf_cap < work_sz) { free(ctx.work_buf); ctx.work_buf = (uint8_t*)malloc(work_sz); // LCOV_EXCL_START if (UNLIKELY(!ctx.work_buf)) { zxc_cctx_free(&ctx); return ZXC_ERROR_MEMORY; } // LCOV_EXCL_STOP ctx.work_buf_cap = work_sz; } // Block decompression loop uint32_t global_hash = 0; while (ip < ip_end) { const size_t rem_src = (size_t)(ip_end - ip); zxc_block_header_t bh; // Read the block header to determine the compressed size if (UNLIKELY(zxc_read_block_header(ip, rem_src, &bh) != ZXC_OK)) { zxc_cctx_free(&ctx); return ZXC_ERROR_BAD_HEADER; } // Handle EOF block separately (not a real chunk to decompress) if (UNLIKELY(bh.block_type == ZXC_BLOCK_EOF)) { // Footer is always the last ZXC_FILE_FOOTER_SIZE bytes of the source, // even when a seek table is inserted between EOF block and footer. // LCOV_EXCL_START if (UNLIKELY(src_size < ZXC_FILE_FOOTER_SIZE)) { zxc_cctx_free(&ctx); return ZXC_ERROR_SRC_TOO_SMALL; } // LCOV_EXCL_STOP const uint8_t* const footer = (const uint8_t*)src + src_size - ZXC_FILE_FOOTER_SIZE; // Validate source size matches what we decompressed const uint64_t stored_size = zxc_le64(footer); if (UNLIKELY(stored_size != (uint64_t)(op - op_start))) { zxc_cctx_free(&ctx); return ZXC_ERROR_CORRUPT_DATA; } // Validate global checksum if enabled and file has checksums if (checksum_enabled && file_has_checksums) { const uint32_t stored_hash = zxc_le32(footer + sizeof(uint64_t)); if (UNLIKELY(stored_hash != global_hash)) { zxc_cctx_free(&ctx); return ZXC_ERROR_BAD_CHECKSUM; } } break; // EOF reached, exit loop } int res; const size_t rem_cap = (size_t)(op_end - op); if (LIKELY(rem_cap >= work_sz)) { // Fast path: decode directly into dst. Cap dst_cap to chunk_size + PAD res = zxc_decompress_chunk_wrapper(&ctx, ip, rem_src, op, work_sz); } else { // Safe path: decode into bounce buffer, then copy exact result. res = zxc_decompress_chunk_wrapper(&ctx, ip, rem_src, ctx.work_buf, ctx.work_buf_cap); if (LIKELY(res > 0)) { // LCOV_EXCL_START if (UNLIKELY((size_t)res > rem_cap)) { zxc_cctx_free(&ctx); return ZXC_ERROR_DST_TOO_SMALL; } // LCOV_EXCL_STOP ZXC_MEMCPY(op, ctx.work_buf, (size_t)res); } } if (UNLIKELY(res < 0)) { zxc_cctx_free(&ctx); return res; } // Update global hash from block checksum if (checksum_enabled && file_has_checksums) { const uint32_t block_hash = zxc_le32(ip + ZXC_BLOCK_HEADER_SIZE + bh.comp_size); global_hash = zxc_hash_combine_rotate(global_hash, block_hash); } ip += ZXC_BLOCK_HEADER_SIZE + bh.comp_size + (file_has_checksums ? ZXC_BLOCK_CHECKSUM_SIZE : 0); op += res; } zxc_cctx_free(&ctx); return (int64_t)(op - op_start); } /** * @brief Reads the decompressed size from a ZXC-compressed buffer. * * The size is stored in the file footer (last @ref ZXC_FILE_FOOTER_SIZE bytes). * * @param[in] src Compressed data. * @param[in] src_size Size of @p src in bytes. * @return Original uncompressed size, or 0 on error. */ uint64_t zxc_get_decompressed_size(const void* src, const size_t src_size) { if (UNLIKELY(src_size < ZXC_FILE_HEADER_SIZE + ZXC_FILE_FOOTER_SIZE)) return 0; const uint8_t* const p = (const uint8_t*)src; if (UNLIKELY(zxc_le32(p) != ZXC_MAGIC_WORD)) return 0; const uint8_t* const footer = p + src_size - ZXC_FILE_FOOTER_SIZE; return zxc_le64(footer); } /* * ============================================================================ * REUSABLE CONTEXT API (Opaque) * ============================================================================ * * Provides heap-allocated, opaque contexts that integrators can reuse across * multiple compress / decompress calls, eliminating per-call malloc/free * overhead. */ /* --- Compression --------------------------------------------------------- */ struct zxc_cctx_s { zxc_cctx_t inner; /* existing internal context */ int initialized; /* 1 if inner has live allocations */ size_t last_block_size; /* block size used for last init */ /* Sticky options (remembered from create or last compress call). */ int stored_level; int stored_checksum; size_t stored_block_size; }; zxc_cctx* zxc_create_cctx(const zxc_compress_opts_t* opts) { zxc_cctx* const cctx = (zxc_cctx*)calloc(1, sizeof(zxc_cctx)); if (UNLIKELY(!cctx)) return NULL; // LCOV_EXCL_LINE /* Resolve and store sticky defaults. */ cctx->stored_level = (opts && opts->level > 0) ? opts->level : ZXC_LEVEL_DEFAULT; cctx->stored_block_size = (opts && opts->block_size > 0) ? opts->block_size : ZXC_BLOCK_SIZE_DEFAULT; cctx->stored_checksum = opts ? opts->checksum_enabled : 0; if (opts) { // LCOV_EXCL_START if (UNLIKELY(!zxc_validate_block_size(cctx->stored_block_size) || zxc_cctx_init(&cctx->inner, cctx->stored_block_size, 1, cctx->stored_level, cctx->stored_checksum) != ZXC_OK)) { free(cctx); return NULL; } // LCOV_EXCL_STOP cctx->last_block_size = cctx->stored_block_size; cctx->initialized = 1; } return cctx; } void zxc_free_cctx(zxc_cctx* cctx) { if (UNLIKELY(!cctx)) return; if (cctx->initialized) zxc_cctx_free(&cctx->inner); free(cctx); } int64_t zxc_compress_cctx(zxc_cctx* cctx, const void* RESTRICT src, const size_t src_size, void* RESTRICT dst, const size_t dst_capacity, const zxc_compress_opts_t* opts) { if (UNLIKELY(!cctx)) return ZXC_ERROR_NULL_INPUT; if (UNLIKELY(!src || !dst || src_size == 0 || dst_capacity == 0)) return ZXC_ERROR_NULL_INPUT; const int checksum_enabled = opts ? opts->checksum_enabled : cctx->stored_checksum; const int level = (opts && opts->level > 0) ? opts->level : cctx->stored_level; const size_t block_size = (opts && opts->block_size > 0) ? opts->block_size : cctx->stored_block_size; cctx->stored_level = level; cctx->stored_block_size = block_size; cctx->stored_checksum = checksum_enabled; if (UNLIKELY(!zxc_validate_block_size(block_size))) return ZXC_ERROR_BAD_BLOCK_SIZE; /* Re-init only when block_size changed (it drives buffer sizes). */ if (UNLIKELY(!cctx->initialized || cctx->last_block_size != block_size)) { if (cctx->initialized) { // LCOV_EXCL_START zxc_cctx_free(&cctx->inner); cctx->initialized = 0; // LCOV_EXCL_STOP } // LCOV_EXCL_START if (UNLIKELY(zxc_cctx_init(&cctx->inner, block_size, 1, level, checksum_enabled) != ZXC_OK)) return ZXC_ERROR_MEMORY; // LCOV_EXCL_STOP cctx->last_block_size = block_size; cctx->initialized = 1; } else { /* Same block_size: update level + checksum without realloc. */ cctx->inner.compression_level = level; cctx->inner.checksum_enabled = checksum_enabled; } zxc_cctx_t* const ctx = &cctx->inner; uint8_t* op = (uint8_t*)dst; const uint8_t* const op_start = op; const uint8_t* const op_end = op + dst_capacity; const uint8_t* const ip = (const uint8_t*)src; uint32_t global_hash = 0; const int h_val = zxc_write_file_header(op, (size_t)(op_end - op), block_size, checksum_enabled); if (UNLIKELY(h_val < 0)) return h_val; // LCOV_EXCL_LINE op += h_val; size_t pos = 0; while (pos < src_size) { const size_t chunk_len = (src_size - pos > block_size) ? block_size : (src_size - pos); const size_t rem_cap = (size_t)(op_end - op); const int res = zxc_compress_chunk_wrapper(ctx, ip + pos, chunk_len, op, rem_cap); if (UNLIKELY(res < 0)) return res; if (checksum_enabled) { if (LIKELY(res >= ZXC_GLOBAL_CHECKSUM_SIZE)) { const uint32_t block_hash = zxc_le32(op + res - ZXC_GLOBAL_CHECKSUM_SIZE); global_hash = zxc_hash_combine_rotate(global_hash, block_hash); } } op += res; pos += chunk_len; } /* EOF block */ const size_t rem_cap = (size_t)(op_end - op); const zxc_block_header_t eof_bh = { .block_type = ZXC_BLOCK_EOF, .block_flags = 0, .reserved = 0, .comp_size = 0}; const int eof_val = zxc_write_block_header(op, rem_cap, &eof_bh); if (UNLIKELY(eof_val < 0)) return eof_val; // LCOV_EXCL_LINE op += eof_val; if (UNLIKELY(rem_cap < (size_t)eof_val + ZXC_FILE_FOOTER_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; // LCOV_EXCL_LINE const int footer_val = zxc_write_file_footer(op, (size_t)(op_end - op), src_size, global_hash, checksum_enabled); if (UNLIKELY(footer_val < 0)) return footer_val; // LCOV_EXCL_LINE op += footer_val; return (int64_t)(op - op_start); } /* --- Decompression ------------------------------------------------------- */ struct zxc_dctx_s { zxc_cctx_t inner; /* reuses the same internal context type */ size_t last_block_size; /* block size from last header parse */ int initialized; /* 1 if inner has live allocations */ }; zxc_dctx* zxc_create_dctx(void) { zxc_dctx* const dctx = (zxc_dctx*)calloc(1, sizeof(zxc_dctx)); return dctx; } void zxc_free_dctx(zxc_dctx* dctx) { if (UNLIKELY(!dctx)) return; if (dctx->initialized) zxc_cctx_free(&dctx->inner); free(dctx); } int64_t zxc_decompress_dctx(zxc_dctx* dctx, const void* RESTRICT src, const size_t src_size, void* RESTRICT dst, const size_t dst_capacity, const zxc_decompress_opts_t* opts) { if (UNLIKELY(!dctx || !src || !dst || src_size < ZXC_FILE_HEADER_SIZE)) return ZXC_ERROR_NULL_INPUT; const int checksum_enabled = opts ? opts->checksum_enabled : 0; const uint8_t* ip = (const uint8_t*)src; const uint8_t* const ip_end = ip + src_size; uint8_t* op = (uint8_t*)dst; const uint8_t* const op_start = op; const uint8_t* const op_end = op + dst_capacity; size_t runtime_chunk_size = 0; int file_has_checksums = 0; uint32_t global_hash = 0; if (UNLIKELY(zxc_read_file_header(ip, src_size, &runtime_chunk_size, &file_has_checksums) != ZXC_OK)) return ZXC_ERROR_BAD_HEADER; /* Re-init only when block size changed. */ if (UNLIKELY(!dctx->initialized || dctx->last_block_size != runtime_chunk_size)) { if (dctx->initialized) { // LCOV_EXCL_START zxc_cctx_free(&dctx->inner); dctx->initialized = 0; // LCOV_EXCL_STOP } // LCOV_EXCL_START if (UNLIKELY(zxc_cctx_init(&dctx->inner, runtime_chunk_size, 0, 0, file_has_checksums && checksum_enabled) != ZXC_OK)) return ZXC_ERROR_MEMORY; // LCOV_EXCL_STOP dctx->last_block_size = runtime_chunk_size; dctx->initialized = 1; } else { dctx->inner.checksum_enabled = file_has_checksums && checksum_enabled; } zxc_cctx_t* const ctx = &dctx->inner; ip += ZXC_FILE_HEADER_SIZE; /* Ensure scratch buffer is large enough. */ const size_t work_sz = runtime_chunk_size + ZXC_PAD_SIZE; if (UNLIKELY(ctx->work_buf_cap < work_sz)) { free(ctx->work_buf); ctx->work_buf = (uint8_t*)malloc(work_sz); if (UNLIKELY(!ctx->work_buf)) return ZXC_ERROR_MEMORY; // LCOV_EXCL_LINE ctx->work_buf_cap = work_sz; } while (ip < ip_end) { const size_t rem_src = (size_t)(ip_end - ip); zxc_block_header_t bh; if (UNLIKELY(zxc_read_block_header(ip, rem_src, &bh) != ZXC_OK)) return ZXC_ERROR_BAD_HEADER; if (UNLIKELY(bh.block_type == ZXC_BLOCK_EOF)) { if (UNLIKELY(rem_src < ZXC_BLOCK_HEADER_SIZE + ZXC_FILE_FOOTER_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; const uint8_t* const footer = ip + ZXC_BLOCK_HEADER_SIZE; const uint64_t stored_size = zxc_le64(footer); if (UNLIKELY(stored_size != (uint64_t)(op - op_start))) return ZXC_ERROR_CORRUPT_DATA; if (checksum_enabled && file_has_checksums) { const uint32_t stored_hash = zxc_le32(footer + sizeof(uint64_t)); if (UNLIKELY(stored_hash != global_hash)) return ZXC_ERROR_BAD_CHECKSUM; } break; } const size_t rem_cap = (size_t)(op_end - op); int res; if (LIKELY(rem_cap >= work_sz)) { // Fast path: decode directly into dst (enough padding for wild copies). res = zxc_decompress_chunk_wrapper(ctx, ip, rem_src, op, rem_cap); } else { // Safe path: decode into bounce buffer, then copy exact result. res = zxc_decompress_chunk_wrapper(ctx, ip, rem_src, ctx->work_buf, ctx->work_buf_cap); if (LIKELY(res > 0)) { if (UNLIKELY((size_t)res > rem_cap)) return ZXC_ERROR_DST_TOO_SMALL; // LCOV_EXCL_LINE ZXC_MEMCPY(op, ctx->work_buf, (size_t)res); } } if (UNLIKELY(res < 0)) return res; if (checksum_enabled && file_has_checksums) { const uint32_t block_hash = zxc_le32(ip + ZXC_BLOCK_HEADER_SIZE + bh.comp_size); global_hash = zxc_hash_combine_rotate(global_hash, block_hash); } ip += ZXC_BLOCK_HEADER_SIZE + bh.comp_size + (file_has_checksums ? ZXC_BLOCK_CHECKSUM_SIZE : 0); op += res; } return (int64_t)(op - op_start); } /* ========================================================================= */ /* Block-Level API (no file framing) */ /* ========================================================================= */ int64_t zxc_compress_block(zxc_cctx* cctx, const void* RESTRICT src, const size_t src_size, void* RESTRICT dst, const size_t dst_capacity, const zxc_compress_opts_t* opts) { if (UNLIKELY(!cctx || !src || !dst || src_size == 0 || dst_capacity == 0)) return ZXC_ERROR_NULL_INPUT; const int checksum_enabled = opts ? opts->checksum_enabled : cctx->stored_checksum; const int level = (opts && opts->level > 0) ? opts->level : cctx->stored_level; /* For block API, block_size == src_size (the caller compresses one block at a time). */ const size_t block_size = (opts && opts->block_size > 0) ? opts->block_size : cctx->stored_block_size; const size_t min_bs = zxc_block_size_ceil(src_size); /* Always ensure internal buffers can hold src_size. */ const size_t effective_block_size = (block_size > min_bs) ? block_size : min_bs; cctx->stored_level = level; cctx->stored_block_size = effective_block_size; cctx->stored_checksum = checksum_enabled; /* Re-init only when block_size changed. */ if (UNLIKELY(!cctx->initialized || cctx->last_block_size != effective_block_size)) { if (cctx->initialized) { // LCOV_EXCL_START zxc_cctx_free(&cctx->inner); cctx->initialized = 0; // LCOV_EXCL_STOP } // LCOV_EXCL_START if (UNLIKELY(zxc_cctx_init(&cctx->inner, effective_block_size, 1, level, checksum_enabled) != ZXC_OK)) return ZXC_ERROR_MEMORY; // LCOV_EXCL_STOP cctx->last_block_size = effective_block_size; cctx->initialized = 1; } else { cctx->inner.compression_level = level; cctx->inner.checksum_enabled = checksum_enabled; } const int res = zxc_compress_chunk_wrapper(&cctx->inner, (const uint8_t*)src, src_size, (uint8_t*)dst, dst_capacity); if (UNLIKELY(res < 0)) return res; return (int64_t)res; } int64_t zxc_decompress_block(zxc_dctx* dctx, const void* RESTRICT src, const size_t src_size, void* RESTRICT dst, const size_t dst_capacity, const zxc_decompress_opts_t* opts) { if (UNLIKELY(!dctx || !src || !dst || src_size < ZXC_BLOCK_HEADER_SIZE || dst_capacity == 0)) return ZXC_ERROR_NULL_INPUT; const int checksum_enabled = opts ? opts->checksum_enabled : 0; /* Derive the block_size from dst_capacity (callers know the original size). */ const size_t block_size = zxc_block_size_ceil(dst_capacity); if (UNLIKELY(!dctx->initialized || dctx->last_block_size != block_size)) { if (dctx->initialized) { zxc_cctx_free(&dctx->inner); dctx->initialized = 0; } // LCOV_EXCL_START if (UNLIKELY(zxc_cctx_init(&dctx->inner, block_size, 0, 0, checksum_enabled) != ZXC_OK)) return ZXC_ERROR_MEMORY; // LCOV_EXCL_STOP dctx->last_block_size = block_size; dctx->initialized = 1; } else { dctx->inner.checksum_enabled = checksum_enabled; } zxc_cctx_t* const ctx = &dctx->inner; /* Ensure scratch buffer for safe-path wild copies. */ const size_t work_sz = block_size + ZXC_PAD_SIZE; if (ctx->work_buf_cap < work_sz) { free(ctx->work_buf); ctx->work_buf = (uint8_t*)malloc(work_sz); if (UNLIKELY(!ctx->work_buf)) return ZXC_ERROR_MEMORY; // LCOV_EXCL_LINE ctx->work_buf_cap = work_sz; } int res; if (LIKELY(dst_capacity >= work_sz)) { res = zxc_decompress_chunk_wrapper(ctx, (const uint8_t*)src, src_size, (uint8_t*)dst, dst_capacity); } else { /* Bounce through work_buf when output can't absorb wild copies. */ res = zxc_decompress_chunk_wrapper(ctx, (const uint8_t*)src, src_size, ctx->work_buf, ctx->work_buf_cap); if (LIKELY(res > 0)) { if (UNLIKELY((size_t)res > dst_capacity)) return ZXC_ERROR_DST_TOO_SMALL; ZXC_MEMCPY(dst, ctx->work_buf, (size_t)res); } } if (UNLIKELY(res < 0)) return res; return (int64_t)res; } /** * @brief Safe-variant block decompressor: accepts dst_capacity == uncompressed_size. * * Router: NUM/RAW blocks (which never wild-write past dst_capacity) are * forwarded to the existing fast path. GLO/GHI blocks use the strict safe * decoder, avoiding the bounce buffer and the +ZXC_DECOMPRESS_TAIL_PAD * requirement of @ref zxc_decompress_block. */ int64_t zxc_decompress_block_safe(zxc_dctx* dctx, const void* RESTRICT src, const size_t src_size, void* RESTRICT dst, const size_t dst_capacity, const zxc_decompress_opts_t* opts) { if (UNLIKELY(!dctx || !src || !dst || src_size < ZXC_BLOCK_HEADER_SIZE || dst_capacity == 0)) return ZXC_ERROR_NULL_INPUT; const uint8_t type = ((const uint8_t*)src)[0]; /* NUM/RAW never wild-write past dst_capacity: route to the existing fast API. */ if (type == ZXC_BLOCK_NUM || type == ZXC_BLOCK_RAW) { return zxc_decompress_block(dctx, src, src_size, dst, dst_capacity, opts); } /* GLO/GHI: use the strict-tail decoder (no bounce buffer required). */ const int checksum_enabled = opts ? opts->checksum_enabled : 0; const size_t block_size = zxc_block_size_ceil(dst_capacity); if (UNLIKELY(!dctx->initialized || dctx->last_block_size != block_size)) { if (dctx->initialized) { zxc_cctx_free(&dctx->inner); dctx->initialized = 0; } // LCOV_EXCL_START if (UNLIKELY(zxc_cctx_init(&dctx->inner, block_size, 0, 0, checksum_enabled) != ZXC_OK)) return ZXC_ERROR_MEMORY; // LCOV_EXCL_STOP dctx->last_block_size = block_size; dctx->initialized = 1; } else { dctx->inner.checksum_enabled = checksum_enabled; } const int res = zxc_decompress_chunk_wrapper_safe_public(&dctx->inner, (const uint8_t*)src, src_size, (uint8_t*)dst, dst_capacity); if (UNLIKELY(res < 0)) return res; return (int64_t)res; } zxc-0.11.0/src/lib/zxc_driver.c000066400000000000000000001147271520102567100162700ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /** * @file zxc_driver.c * @brief Multi-threaded streaming compression / decompression engine. * * Implements a ring-buffer producer-worker-consumer architecture that * parallelises block processing over @c FILE* streams. Also provides * the public @ref zxc_stream_compress, @ref zxc_stream_decompress, and * extended variants with progress callbacks. */ #include #include #include #include "../../include/zxc_buffer.h" #include "../../include/zxc_error.h" #include "../../include/zxc_sans_io.h" #include "../../include/zxc_seekable.h" #include "../../include/zxc_stream.h" #include "zxc_internal.h" /* * ============================================================================ * WINDOWS THREADING EMULATION * ============================================================================ * Maps POSIX pthread calls to Windows Native API (CriticalSection, * ConditionVariable, Threads). Allows the same threading logic to compile on * Linux/macOS and Windows. */ #if defined(_WIN32) #include #include #include #include // Map POSIX file positioning functions to Windows equivalents #define fseeko _fseeki64 #define ftello _ftelli64 // Simple sysconf emulation to get core count static int zxc_get_num_procs(void) { SYSTEM_INFO sysinfo; GetSystemInfo(&sysinfo); return sysinfo.dwNumberOfProcessors; } typedef CRITICAL_SECTION pthread_mutex_t; typedef CONDITION_VARIABLE pthread_cond_t; typedef HANDLE pthread_t; #define pthread_mutex_init(m, a) InitializeCriticalSection(m) #define pthread_mutex_destroy(m) DeleteCriticalSection(m) #define pthread_mutex_lock(m) EnterCriticalSection(m) #define pthread_mutex_unlock(m) LeaveCriticalSection(m) #define pthread_cond_init(c, a) InitializeConditionVariable(c) #define pthread_cond_destroy(c) (void)(0) #define pthread_cond_wait(c, m) SleepConditionVariableCS(c, m, INFINITE) #define pthread_cond_signal(c) WakeConditionVariable(c) #define pthread_cond_broadcast(c) WakeAllConditionVariable(c) typedef struct { void* (*func)(void*); void* arg; } zxc_win_thread_arg_t; static unsigned __stdcall zxc_win_thread_entry(void* p) { zxc_win_thread_arg_t* a = (zxc_win_thread_arg_t*)p; void* (*f)(void*) = a->func; void* arg = a->arg; free(a); f(arg); return 0; } static int pthread_create(pthread_t* thread, const void* attr, void* (*start_routine)(void*), void* arg) { (void)attr; zxc_win_thread_arg_t* wrapper = malloc(sizeof(zxc_win_thread_arg_t)); if (UNLIKELY(!wrapper)) return ZXC_ERROR_MEMORY; wrapper->func = start_routine; wrapper->arg = arg; uintptr_t handle = _beginthreadex(NULL, 0, zxc_win_thread_entry, wrapper, 0, NULL); if (UNLIKELY(handle == 0)) { free(wrapper); return ZXC_ERROR_MEMORY; } *thread = (HANDLE)handle; return 0; } static int pthread_join(pthread_t thread, void** retval) { (void)retval; WaitForSingleObject(thread, INFINITE); CloseHandle(thread); return 0; } #define sysconf(x) zxc_get_num_procs() #define _SC_NPROCESSORS_ONLN 0 #else #include #include #endif /* * ============================================================================ * STREAMING ENGINE (Producer / Worker / Consumer) * ============================================================================ * Implements a Ring Buffer architecture to parallelize block processing. */ /** * @enum job_status_t * @brief Represents the lifecycle states of a processing job within the ring * buffer. * * @var JOB_STATUS_FREE * The job slot is empty and available to be filled with new data by the * writer. * @var JOB_STATUS_FILLED * The job slot has been populated with input data and is ready for * processing by a worker. * @var JOB_STATUS_PROCESSED * The worker has finished processing the data; the result is ready to be * consumed/written out. */ typedef enum { JOB_STATUS_FREE, JOB_STATUS_FILLED, JOB_STATUS_PROCESSED } job_status_t; /** * @struct zxc_stream_job_t * @brief Represents a single unit of work (a chunk of data) to be processed. * * This structure holds the input and output buffers for a specific chunk of * data, along with its processing status. It is padded to align with cache * lines to prevent false sharing in a multi-threaded environment. * * @var zxc_stream_job_t::in_buf * Pointer to the buffer containing raw input data. * @var zxc_stream_job_t::in_cap * The total allocated capacity of the input buffer. * @var zxc_stream_job_t::in_sz * The actual size of the valid data currently in the input buffer. * @var zxc_stream_job_t::out_buf * Pointer to the buffer where processed (compressed/decompressed) data is * stored. * @var zxc_stream_job_t::out_cap * The total allocated capacity of the output buffer. * @var zxc_stream_job_t::result_sz * The actual size of the valid data produced in the output buffer. * @var zxc_stream_job_t::job_id * A unique identifier for the job, often used for ordering or debugging. * @var zxc_stream_job_t::status * The current state of this job (Free, Filled, or Processed). * @var zxc_stream_job_t::pad * Padding bytes to ensure the structure size aligns with typical cache * lines (64 bytes), minimizing cache contention between threads accessing * adjacent jobs. */ typedef struct { uint8_t* in_buf; size_t in_cap, in_sz; uint8_t* out_buf; size_t out_cap, result_sz; int job_id; ZXC_ATOMIC job_status_t status; // Atomic for lock-free status updates char pad[ZXC_CACHE_LINE_SIZE]; // Prevent False Sharing } zxc_stream_job_t; /** * @typedef zxc_chunk_processor_t * @brief Function pointer type for processing a chunk of data. * * This type defines the signature for internal functions responsible for * processing (compressing or transforming) a specific chunk of input data. * * @param ctx Pointer to the compression context containing state and * configuration. * @param in Pointer to the input data buffer. * @param in_sz Size of the input data in bytes. * @param out Pointer to the output buffer where processed data will be * written. * @param out_cap Capacity of the output buffer in bytes. * * @return The number of bytes written to the output buffer on success, or a * negative error code on failure. */ typedef int (*zxc_chunk_processor_t)(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT in, const size_t in_sz, uint8_t* RESTRICT out, const size_t out_cap); /** * @struct zxc_stream_ctx_t * @brief The main context structure managing the streaming * compression/decompression state. * * This structure orchestrates the producer-consumer workflow. It manages the * ring buffer of jobs, the worker queue, synchronization primitives (mutexes * and condition variables), and configuration settings for the compression * algorithm. * * @var zxc_stream_ctx_t::jobs * Array of job structures acting as the ring buffer. * @var zxc_stream_ctx_t::ring_size * The total number of slots in the jobs array. * @var zxc_stream_ctx_t::worker_queue * A circular queue containing indices of jobs ready to be picked up by * worker threads. * @var zxc_stream_ctx_t::wq_head * Index of the head of the worker queue (where workers take jobs). * @var zxc_stream_ctx_t::wq_tail * Index of the tail of the worker queue (where the writer adds jobs). * @var zxc_stream_ctx_t::wq_count * Current number of items in the worker queue. * @var zxc_stream_ctx_t::lock * Mutex used to protect access to shared resources (queue indices, status * changes). * @var zxc_stream_ctx_t::cond_reader * Condition variable to signal the output thread (reader) that processed * data is available. * @var zxc_stream_ctx_t::cond_worker * Condition variable to signal worker threads that new work is available. * @var zxc_stream_ctx_t::cond_writer * Condition variable to signal the input thread (writer) that job slots * are free. * @var zxc_stream_ctx_t::shutdown_workers * Flag indicating that worker threads should terminate. * @var zxc_stream_ctx_t::compression_mode * Indicates the operation mode (e.g., compression or decompression). * @var zxc_stream_ctx_t::io_error * Atomic flag to signal if an I/O error occurred during processing. * @var zxc_stream_ctx_t::processor * Function pointer or object responsible for the actual chunk processing * logic. * @var zxc_stream_ctx_t::write_idx * The index of the next job slot to be written to by the main thread. * @var zxc_stream_ctx_t::compression_level * The configured level of compression (trading off speed vs. ratio). * @var zxc_stream_ctx_t::chunk_size * The size of each data chunk to be processed. * @var zxc_stream_ctx_t::checksum_enabled * Flag indicating whether checksum verification/generation is active. * @var zxc_stream_ctx_t::file_has_checksum * Flag indicating whether the input file includes checksums. * @var zxc_stream_ctx_t::progress_cb * Optional callback function for reporting progress during processing. * @var zxc_stream_ctx_t::progress_user_data * User data pointer to be passed to the progress callback function. * @var zxc_stream_ctx_t::total_input_bytes * Total size of the input data in bytes, used for progress tracking. */ typedef struct { zxc_stream_job_t* jobs; size_t ring_size; int* worker_queue; int wq_head, wq_tail, wq_count; pthread_mutex_t lock; pthread_cond_t cond_reader, cond_worker, cond_writer; int shutdown_workers; int compression_mode; ZXC_ATOMIC int io_error; zxc_chunk_processor_t processor; int write_idx; int compression_level; size_t chunk_size; int checksum_enabled; int file_has_checksum; zxc_progress_callback_t progress_cb; void* progress_user_data; uint64_t total_input_bytes; } zxc_stream_ctx_t; /** * @struct writer_args_t * @brief Structure containing arguments for the writer callback function. * * This structure is used to pass necessary context and state information * to the function responsible for writing compressed or decompressed data * to a file stream. * * @var writer_args_t::ctx * Pointer to the ZXC stream context, holding the state of the * compression/decompression stream. * * @var writer_args_t::f * Pointer to the output file stream where data will be written. * * @var writer_args_t::total_bytes * Accumulator for the total number of bytes written to the file so far. * * @var writer_args_t::global_hash * The global hash accumulated during processing. * * @var writer_args_t::bytes_processed * The number of bytes processed so far, used for progress reporting. * * @var writer_args_t::seek_comp * Array of compressed block sizes for seek table construction. * * @var writer_args_t::seek_count * Number of entries in the seek table. * * @var writer_args_t::seek_cap * Capacity of the seek table array. */ typedef struct { zxc_stream_ctx_t* ctx; FILE* f; int64_t total_bytes; uint32_t global_hash; uint64_t bytes_processed; // For progress callback uint32_t* seek_comp; uint32_t seek_count; uint32_t seek_cap; } writer_args_t; /** * @brief Worker thread function for parallel stream processing. * * This function serves as the entry point for worker threads in the ZXC * streaming compression/decompression context. It continuously retrieves jobs * from a shared work queue, processes them using a thread-local compression * context (`zxc_cctx_t`), and signals the writer thread upon completion. * * **Worker Lifecycle & Synchronization:** * 1. **Initialization:** Allocates a thread-local `zxc_cctx_t` to avoid lock * contention during compression/decompression. * 2. **Wait Loop:** Uses `pthread_cond_wait` on `cond_worker` to sleep until a * job is available in the `worker_queue`. * 3. **Job Retrieval:** Dequeues a job ID from the ring buffer. The * `worker_queue` acts as a load balancer. * 4. **Processing:** Calls `ctx->processor` (the compression/decompression * function) on the job's data. This is the CPU-intensive part and runs in * parallel. * 5. **Completion:** Updates `job->status` to `JOB_STATUS_PROCESSED`. * 6. **Signaling:** If the processed job is the *next* one expected by the * writer * (`jid == ctx->write_idx`), it signals `cond_writer`. This optimization * prevents unnecessary wake-ups of the writer thread for out-of-order * completions. * * @param[in] arg A pointer to the shared stream context (`zxc_stream_ctx_t`). * @return Always returns NULL. */ static void* zxc_stream_worker(void* arg) { zxc_stream_ctx_t* const ctx = (zxc_stream_ctx_t*)arg; zxc_cctx_t cctx; const int unified_chk = (ctx->compression_mode == 1) ? ctx->checksum_enabled : (ctx->file_has_checksum && ctx->checksum_enabled); if (zxc_cctx_init(&cctx, ctx->chunk_size, ctx->compression_mode, ctx->compression_level, unified_chk) != ZXC_OK) { // LCOV_EXCL_START zxc_cctx_free(&cctx); pthread_mutex_lock(&ctx->lock); ctx->io_error = 1; pthread_cond_broadcast(&ctx->cond_writer); pthread_cond_broadcast(&ctx->cond_reader); pthread_mutex_unlock(&ctx->lock); return NULL; // LCOV_EXCL_STOP } cctx.compression_level = ctx->compression_level; while (1) { zxc_stream_job_t* job = NULL; pthread_mutex_lock(&ctx->lock); while (ctx->wq_count == 0 && !ctx->shutdown_workers) { pthread_cond_wait(&ctx->cond_worker, &ctx->lock); } if (ctx->shutdown_workers && ctx->wq_count == 0) { pthread_mutex_unlock(&ctx->lock); break; } const int jid = ctx->worker_queue[ctx->wq_tail]; ctx->wq_tail = (ctx->wq_tail + 1) % ctx->ring_size; ctx->wq_count--; job = &ctx->jobs[jid]; pthread_mutex_unlock(&ctx->lock); const int res = ctx->processor(&cctx, job->in_buf, job->in_sz, job->out_buf, job->out_cap); pthread_mutex_lock(&ctx->lock); job->result_sz = UNLIKELY(res < 0) ? 0 : (size_t)res; job->status = JOB_STATUS_PROCESSED; if (UNLIKELY(res < 0)) { ctx->io_error = 1; pthread_cond_broadcast(&ctx->cond_writer); pthread_cond_broadcast(&ctx->cond_reader); } else if (jid == ctx->write_idx) { pthread_cond_signal(&ctx->cond_writer); } pthread_mutex_unlock(&ctx->lock); } zxc_cctx_free(&cctx); return NULL; } /** * @brief Asynchronous writer thread function. * * This function runs as a separate thread responsible for writing processed * data chunks to the output file. It operates on a ring buffer of jobs shared * with the reader and worker threads. * * **Ordering Enforcement:** * The writer MUST write blocks in the exact order they were read. Even if * worker threads finish jobs out of order (e.g., job 2 finishes before job 1), * the writer waits for `ctx->write_idx` (job 1) to be `JOB_STATUS_PROCESSED`. * * **Workflow:** * 1. **Wait:** Sleeps on `cond_writer` until the job at `ctx->write_idx` is * ready. * 2. **Write:** Writes the `out_buf` to the file. * 3. **Release:** Sets the job status to `JOB_STATUS_FREE` and signals * `cond_reader`, allowing the main thread to reuse this slot for new input. * 4. **Advance:** Increments `ctx->write_idx` to wait for the next sequential * block. * * @param[in] arg Pointer to a `writer_args_t` structure containing the stream * context, the output file handle, and a counter for total bytes written. * @return Always returns NULL. */ static void* zxc_async_writer(void* arg) { writer_args_t* const args = (writer_args_t*)arg; zxc_stream_ctx_t* const ctx = args->ctx; while (1) { zxc_stream_job_t* const job = &ctx->jobs[ctx->write_idx]; pthread_mutex_lock(&ctx->lock); while (job->status != JOB_STATUS_PROCESSED && !ctx->io_error) pthread_cond_wait(&ctx->cond_writer, &ctx->lock); const size_t result_sz = job->result_sz; const size_t in_sz = job->in_sz; pthread_mutex_unlock(&ctx->lock); if (result_sz == (size_t)-1) break; if (args->f && result_sz > 0) { if (fwrite(job->out_buf, 1, result_sz, args->f) != result_sz) { pthread_mutex_lock(&ctx->lock); ctx->io_error = 1; pthread_cond_signal(&ctx->cond_reader); pthread_mutex_unlock(&ctx->lock); } else if (ctx->checksum_enabled && ctx->compression_mode == 1) { // Update Global Hash (Rotation + XOR) if (LIKELY(result_sz >= ZXC_GLOBAL_CHECKSUM_SIZE)) { uint32_t block_hash = zxc_le32(job->out_buf + result_sz - ZXC_GLOBAL_CHECKSUM_SIZE); args->global_hash = zxc_hash_combine_rotate(args->global_hash, block_hash); } } } if (UNLIKELY(ctx->io_error)) { pthread_mutex_lock(&ctx->lock); job->status = JOB_STATUS_FREE; pthread_cond_signal(&ctx->cond_reader); pthread_mutex_unlock(&ctx->lock); break; } args->total_bytes += (int64_t)result_sz; /* Seekable: record compressed block size */ if (args->seek_comp && ctx->compression_mode == 1) { if (UNLIKELY(args->seek_count >= args->seek_cap)) { args->seek_cap = args->seek_cap * 2; uint32_t* nc = (uint32_t*)realloc(args->seek_comp, args->seek_cap * sizeof(uint32_t)); // LCOV_EXCL_START if (UNLIKELY(!nc)) { pthread_mutex_lock(&ctx->lock); ctx->io_error = 1; job->status = JOB_STATUS_FREE; pthread_cond_signal(&ctx->cond_reader); pthread_mutex_unlock(&ctx->lock); break; } // LCOV_EXCL_STOP args->seek_comp = nc; } args->seek_comp[args->seek_count++] = (uint32_t)result_sz; } // Update progress callback if (ctx->progress_cb) { // LCOV_EXCL_START args->bytes_processed += ctx->compression_mode == 1 ? in_sz : result_sz; ctx->progress_cb(args->bytes_processed, ctx->total_input_bytes, ctx->progress_user_data); // LCOV_EXCL_STOP } pthread_mutex_lock(&ctx->lock); job->status = JOB_STATUS_FREE; ctx->write_idx = (ctx->write_idx + 1) % ctx->ring_size; pthread_cond_signal(&ctx->cond_reader); pthread_mutex_unlock(&ctx->lock); } return NULL; } /** * @brief Orchestrates the multithreaded streaming compression or decompression * engine. * * This function initializes the stream context, allocates the necessary ring * buffer memory for jobs and I/O buffers, and spawns the worker threads and the * asynchronous writer thread. It acts as the main "producer" (reader) loop. * * **Architecture: Producer-Consumer with Ring Buffer** * - **Ring Buffer:** A fixed-size array of `zxc_stream_job_t` structures. * - **Producer (Main Thread):** Reads chunks from `f_in` and fills "Free" slots * in the ring buffer. It blocks if no slots are free (backpressure). * - **Workers:** Pick up "Filled" jobs from a queue, process them, and mark * them as "Processed". * - **Consumer (Writer Thread):** Waits for the *next sequential* job to be * "Processed", writes it to `f_out`, and marks the slot as "Free". * * **Double-Buffering & Zero-Copy:** * We allocate `alloc_in` and `alloc_out` buffers for each job. The reader reads * directly into `in_buf`, and the writer writes directly from `out_buf`, * minimizing memory copies. * * @param[in] f_in Pointer to the input file stream (source). * @param[out] f_out Pointer to the output file stream (destination). * @param[in] n_threads Number of worker threads to spawn. If set to 0 or less, the * function automatically detects the number of online processors. * @param[in] mode Operation mode: 1 for compression, 0 for decompression. * @param[in] level Compression level to be applied (relevant for compression * mode). * @param[in] checksum_enabled Flag indicating whether to enable checksum * generation/verification. * @param[in] func Function pointer to the chunk processor (compression or * decompression logic). * * @return The total number of bytes written to the output stream on success, or * -1 if an initialization or I/O error occurred. */ static int64_t zxc_stream_engine_run(FILE* f_in, FILE* f_out, const int n_threads, const int mode, const int level, const size_t block_size, const int checksum_enabled, const int seekable, zxc_chunk_processor_t func, zxc_progress_callback_t progress_cb, void* user_data) { zxc_stream_ctx_t ctx; ZXC_MEMSET(&ctx, 0, sizeof(ctx)); size_t runtime_chunk_sz = (block_size > 0) ? block_size : ZXC_BLOCK_SIZE_DEFAULT; int file_has_chk = 0; // Try to get input file size for progress tracking (compression mode only) // For decompression, the CLI precomputes the size and passes it via user_data uint64_t total_file_size = 0; if (mode == 1 && progress_cb) { // LCOV_EXCL_START const long long saved_pos = ftello(f_in); if (saved_pos >= 0) { if (fseeko(f_in, 0, SEEK_END) == 0) { const long long size = ftello(f_in); if (size > 0) total_file_size = (uint64_t)size; fseeko(f_in, saved_pos, SEEK_SET); } } // LCOV_EXCL_STOP } if (mode == 0) { // Decompression Mode: Read and validate file header uint8_t h[ZXC_FILE_HEADER_SIZE]; if (UNLIKELY(fread(h, 1, ZXC_FILE_HEADER_SIZE, f_in) != ZXC_FILE_HEADER_SIZE || zxc_read_file_header(h, ZXC_FILE_HEADER_SIZE, &runtime_chunk_sz, &file_has_chk) != ZXC_OK)) return ZXC_ERROR_BAD_HEADER; } int num_threads = (n_threads > 0) ? n_threads : (int)sysconf(_SC_NPROCESSORS_ONLN); if (num_threads > ZXC_MAX_THREADS) num_threads = ZXC_MAX_THREADS; // Reserve 1 thread for Writer/Reader overhead if possible const int num_workers = (num_threads > 1) ? num_threads - 1 : 1; ctx.compression_mode = mode; ctx.processor = func; ctx.io_error = 0; ctx.compression_level = level; ctx.ring_size = (size_t)num_workers * 4U; ctx.chunk_size = runtime_chunk_sz; ctx.checksum_enabled = checksum_enabled; ctx.file_has_checksum = mode == 1 ? checksum_enabled : file_has_chk; ctx.progress_cb = progress_cb; ctx.progress_user_data = user_data; ctx.total_input_bytes = total_file_size; uint32_t d_global_hash = 0; const uint64_t max_out = zxc_compress_bound(runtime_chunk_sz); const size_t raw_alloc_in = (size_t)(((mode) ? runtime_chunk_sz : max_out) + ZXC_PAD_SIZE); const size_t alloc_in = (raw_alloc_in + ZXC_ALIGNMENT_MASK) & ~ZXC_ALIGNMENT_MASK; const size_t raw_alloc_out = (size_t)(((mode) ? max_out : runtime_chunk_sz) + ZXC_PAD_SIZE); const size_t alloc_out = (raw_alloc_out + ZXC_ALIGNMENT_MASK) & ~ZXC_ALIGNMENT_MASK; const size_t per_job_sz = sizeof(zxc_stream_job_t) + sizeof(int) + alloc_in + alloc_out; const size_t alloc_size = ctx.ring_size * per_job_sz; uint8_t* const mem_block = zxc_aligned_malloc(alloc_size, ZXC_CACHE_LINE_SIZE); if (UNLIKELY(!mem_block || per_job_sz > SIZE_MAX / ctx.ring_size)) { // LCOV_EXCL_START zxc_aligned_free(mem_block); return ZXC_ERROR_MEMORY; // LCOV_EXCL_STOP } uint8_t* ptr = mem_block; ctx.jobs = (zxc_stream_job_t*)ptr; ptr += ctx.ring_size * sizeof(zxc_stream_job_t); ctx.worker_queue = (int*)ptr; ptr += ctx.ring_size * sizeof(int); uint8_t* buf_in = ptr; ptr += ctx.ring_size * alloc_in; uint8_t* buf_out = ptr; ZXC_MEMSET(mem_block, 0, alloc_size); for (size_t i = 0; i < ctx.ring_size; i++) { ctx.jobs[i].job_id = (int)i; ctx.jobs[i].status = JOB_STATUS_FREE; ctx.jobs[i].in_buf = buf_in + (i * alloc_in); ctx.jobs[i].in_cap = alloc_in - ZXC_PAD_SIZE; ctx.jobs[i].in_sz = 0; ctx.jobs[i].out_buf = buf_out + (i * alloc_out); ctx.jobs[i].out_cap = alloc_out - ZXC_PAD_SIZE; ctx.jobs[i].result_sz = 0; } pthread_mutex_init(&ctx.lock, NULL); pthread_cond_init(&ctx.cond_reader, NULL); pthread_cond_init(&ctx.cond_worker, NULL); pthread_cond_init(&ctx.cond_writer, NULL); pthread_t* const workers = malloc((size_t)num_workers * sizeof(pthread_t)); if (UNLIKELY(!workers)) { // LCOV_EXCL_START zxc_aligned_free(mem_block); return ZXC_ERROR_MEMORY; // LCOV_EXCL_STOP } int started_workers = 0; for (int i = 0; i < num_workers; i++) { if (UNLIKELY(pthread_create(&workers[i], NULL, zxc_stream_worker, &ctx) != 0)) break; started_workers++; } if (UNLIKELY(started_workers == 0)) { // LCOV_EXCL_START pthread_cond_destroy(&ctx.cond_writer); pthread_cond_destroy(&ctx.cond_worker); pthread_cond_destroy(&ctx.cond_reader); pthread_mutex_destroy(&ctx.lock); free(workers); zxc_aligned_free(mem_block); return ZXC_ERROR_MEMORY; // LCOV_EXCL_STOP } writer_args_t w_args = {&ctx, f_out, 0, 0, 0, NULL, 0, 0}; /* Seekable: allocate initial block-size tracking array */ if (mode == 1 && seekable) { w_args.seek_cap = 64; w_args.seek_comp = (uint32_t*)malloc(w_args.seek_cap * sizeof(uint32_t)); // LCOV_EXCL_START if (UNLIKELY(!w_args.seek_comp)) { pthread_mutex_lock(&ctx.lock); ctx.shutdown_workers = 1; pthread_cond_broadcast(&ctx.cond_worker); pthread_mutex_unlock(&ctx.lock); for (int i = 0; i < started_workers; i++) pthread_join(workers[i], NULL); pthread_cond_destroy(&ctx.cond_writer); pthread_cond_destroy(&ctx.cond_worker); pthread_cond_destroy(&ctx.cond_reader); pthread_mutex_destroy(&ctx.lock); free(workers); zxc_aligned_free(mem_block); return ZXC_ERROR_MEMORY; } // LCOV_EXCL_STOP } if (mode == 1 && f_out) { uint8_t h[ZXC_FILE_HEADER_SIZE]; zxc_write_file_header(h, ZXC_FILE_HEADER_SIZE, runtime_chunk_sz, checksum_enabled); if (UNLIKELY(fwrite(h, 1, ZXC_FILE_HEADER_SIZE, f_out) != ZXC_FILE_HEADER_SIZE)) ctx.io_error = 1; w_args.total_bytes = ZXC_FILE_HEADER_SIZE; } pthread_t writer_th; if (UNLIKELY(pthread_create(&writer_th, NULL, zxc_async_writer, &w_args) != 0)) { // LCOV_EXCL_START pthread_mutex_lock(&ctx.lock); ctx.shutdown_workers = 1; pthread_cond_broadcast(&ctx.cond_worker); pthread_mutex_unlock(&ctx.lock); for (int i = 0; i < started_workers; i++) pthread_join(workers[i], NULL); pthread_cond_destroy(&ctx.cond_writer); pthread_cond_destroy(&ctx.cond_worker); pthread_cond_destroy(&ctx.cond_reader); pthread_mutex_destroy(&ctx.lock); free(workers); zxc_aligned_free(mem_block); return ZXC_ERROR_MEMORY; // LCOV_EXCL_STOP } int read_idx = 0; int read_eof = 0; uint64_t total_src_bytes = 0; // Reader Loop: Reads from file, prepares jobs, pushes to worker queue. while (!read_eof && !ctx.io_error) { zxc_stream_job_t* const job = &ctx.jobs[read_idx]; pthread_mutex_lock(&ctx.lock); while (job->status != JOB_STATUS_FREE && !ctx.io_error) pthread_cond_wait(&ctx.cond_reader, &ctx.lock); pthread_mutex_unlock(&ctx.lock); if (UNLIKELY(ctx.io_error)) break; size_t read_sz = 0; if (mode == 1) { read_sz = fread(job->in_buf, 1, runtime_chunk_sz, f_in); total_src_bytes += read_sz; if (UNLIKELY(read_sz == 0)) read_eof = 1; } else { uint8_t bh_buf[ZXC_BLOCK_HEADER_SIZE]; size_t h_read = fread(bh_buf, 1, ZXC_BLOCK_HEADER_SIZE, f_in); if (UNLIKELY(h_read < ZXC_BLOCK_HEADER_SIZE)) { read_eof = 1; } else { zxc_block_header_t bh; if (UNLIKELY(zxc_read_block_header(bh_buf, ZXC_BLOCK_HEADER_SIZE, &bh) != ZXC_OK)) { read_eof = 1; goto _job_prepared; } if (bh.block_type == ZXC_BLOCK_EOF) { if (UNLIKELY(bh.comp_size != 0)) { ctx.io_error = 1; goto _job_prepared; } read_eof = 1; read_sz = 0; goto _job_prepared; } const int has_crc = ctx.file_has_checksum; const size_t checksum_sz = (has_crc ? ZXC_BLOCK_CHECKSUM_SIZE : 0); const size_t body_total = bh.comp_size + checksum_sz; const size_t total_len = ZXC_BLOCK_HEADER_SIZE + body_total; if (UNLIKELY(total_len > job->in_cap)) { ctx.io_error = 1; break; } ZXC_MEMCPY(job->in_buf, bh_buf, ZXC_BLOCK_HEADER_SIZE); // Single fread for body + checksum (reduces syscalls) const size_t body_read = fread(job->in_buf + ZXC_BLOCK_HEADER_SIZE, 1, body_total, f_in); if (UNLIKELY(body_read != body_total)) { ctx.io_error = 1; break; } else if (has_crc) { // Update Global Hash for Decompression const uint32_t b_crc = zxc_le32(job->in_buf + ZXC_BLOCK_HEADER_SIZE + bh.comp_size); d_global_hash = zxc_hash_combine_rotate(d_global_hash, b_crc); } read_sz = ZXC_BLOCK_HEADER_SIZE + body_read; } } _job_prepared: if (UNLIKELY(read_eof && read_sz == 0)) break; job->in_sz = read_sz; pthread_mutex_lock(&ctx.lock); job->status = JOB_STATUS_FILLED; ctx.worker_queue[ctx.wq_head] = read_idx; ctx.wq_head = (ctx.wq_head + 1) % ctx.ring_size; ctx.wq_count++; read_idx = (read_idx + 1) % ctx.ring_size; pthread_cond_signal(&ctx.cond_worker); pthread_mutex_unlock(&ctx.lock); if (UNLIKELY(read_sz < runtime_chunk_sz && mode == 1)) read_eof = 1; } zxc_stream_job_t* const end_job = &ctx.jobs[read_idx]; pthread_mutex_lock(&ctx.lock); while (end_job->status != JOB_STATUS_FREE && !ctx.io_error) pthread_cond_wait(&ctx.cond_reader, &ctx.lock); end_job->result_sz = (size_t)-1; end_job->status = JOB_STATUS_PROCESSED; pthread_cond_broadcast(&ctx.cond_writer); pthread_mutex_unlock(&ctx.lock); pthread_join(writer_th, NULL); pthread_mutex_lock(&ctx.lock); ctx.shutdown_workers = 1; pthread_cond_broadcast(&ctx.cond_worker); pthread_mutex_unlock(&ctx.lock); for (int i = 0; i < started_workers; i++) pthread_join(workers[i], NULL); pthread_cond_destroy(&ctx.cond_writer); pthread_cond_destroy(&ctx.cond_worker); pthread_cond_destroy(&ctx.cond_reader); pthread_mutex_destroy(&ctx.lock); // Write EOF Block + optional Seek Table + Footer if compression and no error if (mode == 1 && !ctx.io_error && w_args.total_bytes >= 0) { /* EOF block */ uint8_t eof_buf[ZXC_BLOCK_HEADER_SIZE]; const zxc_block_header_t eof_bh = { .block_type = ZXC_BLOCK_EOF, .block_flags = 0, .reserved = 0, .comp_size = 0}; zxc_write_block_header(eof_buf, ZXC_BLOCK_HEADER_SIZE, &eof_bh); if (UNLIKELY(f_out && fwrite(eof_buf, 1, ZXC_BLOCK_HEADER_SIZE, f_out) != ZXC_BLOCK_HEADER_SIZE)) ctx.io_error = 1; else w_args.total_bytes += ZXC_BLOCK_HEADER_SIZE; /* Seekable: write SEK block between EOF and footer */ if (!ctx.io_error && w_args.seek_comp && w_args.seek_count > 0) { const size_t st_size = zxc_seek_table_size(w_args.seek_count); uint8_t* const st_buf = (uint8_t*)malloc(st_size); if (st_buf) { const int64_t st_val = zxc_write_seek_table(st_buf, st_size, w_args.seek_comp, w_args.seek_count); if (st_val > 0 && f_out && fwrite(st_buf, 1, (size_t)st_val, f_out) == (size_t)st_val) w_args.total_bytes += st_val; free(st_buf); } } /* Footer */ uint8_t footer_buf[ZXC_FILE_FOOTER_SIZE]; zxc_write_file_footer(footer_buf, ZXC_FILE_FOOTER_SIZE, total_src_bytes, w_args.global_hash, checksum_enabled); if (UNLIKELY(f_out && fwrite(footer_buf, 1, ZXC_FILE_FOOTER_SIZE, f_out) != ZXC_FILE_FOOTER_SIZE)) ctx.io_error = 1; else w_args.total_bytes += ZXC_FILE_FOOTER_SIZE; } else if (mode == 0 && !ctx.io_error) { /* * After the EOF block, the stream may contain: * (a) [FOOTER 12B] - no seekable table * (b) [SEK header 8B] [payload] [FOOTER 12B] - seekable archive */ uint8_t peek_buf[ZXC_BLOCK_HEADER_SIZE]; uint8_t footer[ZXC_FILE_FOOTER_SIZE]; if (UNLIKELY(fread(peek_buf, 1, ZXC_BLOCK_HEADER_SIZE, f_in) != ZXC_BLOCK_HEADER_SIZE)) { ctx.io_error = 1; } else { zxc_block_header_t peek_bh; const int is_sek = (zxc_read_block_header(peek_buf, ZXC_BLOCK_HEADER_SIZE, &peek_bh) == ZXC_OK && peek_bh.block_type == ZXC_BLOCK_SEK); if (is_sek) { /* Drain the SEK payload (read + discard) */ size_t remaining = (size_t)peek_bh.comp_size; uint8_t discard[512]; while (remaining > 0 && !ctx.io_error) { const size_t chunk = remaining < sizeof(discard) ? remaining : sizeof(discard); if (UNLIKELY(fread(discard, 1, chunk, f_in) != chunk)) ctx.io_error = 1; remaining -= chunk; } /* Read full 12-byte footer */ if (!ctx.io_error && UNLIKELY(fread(footer, 1, ZXC_FILE_FOOTER_SIZE, f_in) != ZXC_FILE_FOOTER_SIZE)) ctx.io_error = 1; } else { /* peek_buf contains the first 8 bytes of the 12-byte footer. * Read the remaining 4 bytes and assemble. */ ZXC_MEMCPY(footer, peek_buf, ZXC_BLOCK_HEADER_SIZE); const size_t tail = ZXC_FILE_FOOTER_SIZE - ZXC_BLOCK_HEADER_SIZE; /* 4 */ if (UNLIKELY(fread(footer + ZXC_BLOCK_HEADER_SIZE, 1, tail, f_in) != tail)) ctx.io_error = 1; } } /* Verify Footer Content: Source Size and Global Checksum */ if (!ctx.io_error) { int valid = (zxc_le64(footer) == (uint64_t)w_args.total_bytes); if (valid && checksum_enabled && ctx.file_has_checksum) valid = (zxc_le32(footer + sizeof(uint64_t)) == d_global_hash); if (UNLIKELY(!valid)) ctx.io_error = 1; } } free(w_args.seek_comp); free(workers); zxc_aligned_free(mem_block); if (UNLIKELY(ctx.io_error)) return ZXC_ERROR_IO; return w_args.total_bytes; } int64_t zxc_stream_compress(FILE* f_in, FILE* f_out, const zxc_compress_opts_t* opts) { if (UNLIKELY(!f_in)) return ZXC_ERROR_NULL_INPUT; const int n_threads = opts ? opts->n_threads : 0; const int checksum_enabled = opts ? opts->checksum_enabled : 0; const int seekable = opts ? opts->seekable : 0; const int level = (opts && opts->level > 0) ? opts->level : ZXC_LEVEL_DEFAULT; const size_t block_size = (opts && opts->block_size > 0) ? opts->block_size : ZXC_BLOCK_SIZE_DEFAULT; zxc_progress_callback_t cb = opts ? opts->progress_cb : NULL; void* ud = opts ? opts->user_data : NULL; if (UNLIKELY(!zxc_validate_block_size(block_size))) return ZXC_ERROR_BAD_BLOCK_SIZE; return zxc_stream_engine_run(f_in, f_out, n_threads, 1, level, block_size, checksum_enabled, seekable, zxc_compress_chunk_wrapper, cb, ud); } int64_t zxc_stream_decompress(FILE* f_in, FILE* f_out, const zxc_decompress_opts_t* opts) { if (UNLIKELY(!f_in)) return ZXC_ERROR_NULL_INPUT; const int n_threads = opts ? opts->n_threads : 0; const int checksum_enabled = opts ? opts->checksum_enabled : 0; zxc_progress_callback_t cb = opts ? opts->progress_cb : NULL; void* ud = opts ? opts->user_data : NULL; return zxc_stream_engine_run(f_in, f_out, n_threads, 0, 0, 0, checksum_enabled, 0, (zxc_chunk_processor_t)zxc_decompress_chunk_wrapper, cb, ud); } int64_t zxc_stream_get_decompressed_size(FILE* f_in) { if (UNLIKELY(!f_in)) return ZXC_ERROR_NULL_INPUT; const long long saved_pos = ftello(f_in); if (UNLIKELY(saved_pos < 0)) return ZXC_ERROR_IO; // Get file size if (fseeko(f_in, 0, SEEK_END) != 0) return ZXC_ERROR_IO; const long long file_size = ftello(f_in); if (UNLIKELY(file_size < (long long)(ZXC_FILE_HEADER_SIZE + ZXC_FILE_FOOTER_SIZE))) { fseeko(f_in, saved_pos, SEEK_SET); return ZXC_ERROR_SRC_TOO_SMALL; } uint8_t header[ZXC_FILE_HEADER_SIZE]; if (UNLIKELY(fseeko(f_in, 0, SEEK_SET) != 0 || fread(header, 1, ZXC_FILE_HEADER_SIZE, f_in) != ZXC_FILE_HEADER_SIZE)) { fseeko(f_in, saved_pos, SEEK_SET); return ZXC_ERROR_IO; } if (UNLIKELY(zxc_le32(header) != ZXC_MAGIC_WORD)) { fseeko(f_in, saved_pos, SEEK_SET); return ZXC_ERROR_BAD_MAGIC; } uint8_t footer[ZXC_FILE_FOOTER_SIZE]; if (UNLIKELY(fseeko(f_in, file_size - ZXC_FILE_FOOTER_SIZE, SEEK_SET) != 0 || fread(footer, 1, ZXC_FILE_FOOTER_SIZE, f_in) != ZXC_FILE_FOOTER_SIZE)) { fseeko(f_in, saved_pos, SEEK_SET); return ZXC_ERROR_IO; } fseeko(f_in, saved_pos, SEEK_SET); return (int64_t)zxc_le64(footer); } zxc-0.11.0/src/lib/zxc_huffman.c000066400000000000000000001004541520102567100164110ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause * */ /** * @file zxc_huffman.c * @brief Canonical, length-limited (ZXC_HUF_MAX_CODE_LEN) Huffman codec for the GLO literal * * Canonical, length-limited (ZXC_HUF_MAX_CODE_LEN) Huffman codec for the GLO literal * stream at compression level >= 6. Codes are emitted LSB-first; the * decoder uses a 2048-entry multi-symbol lookup table (11-bit lookup, * 1 or 2 symbols per lookup depending on the cumulative code length) * and a 4-way interleaved hot loop. Public declarations live in * zxc_internal.h; the rest is private to this translation unit. */ /* * Function Multi-Versioning Support * If ZXC_FUNCTION_SUFFIX is defined (e.g. _avx2, _neon), rename the public * entry points so each variant TU produces its own copy under a unique symbol * (e.g. zxc_huf_decode_section_avx2). The runtime dispatcher in * zxc_compress.c / zxc_decompress.c routes to the matching variant. * * The defines sit before zxc_internal.h so the header's prototypes are * rewritten with the same suffix as the definitions below. */ #ifdef ZXC_FUNCTION_SUFFIX #define ZXC_CAT_IMPL(x, y) x##y #define ZXC_CAT(x, y) ZXC_CAT_IMPL(x, y) #define zxc_huf_build_code_lengths ZXC_CAT(zxc_huf_build_code_lengths, ZXC_FUNCTION_SUFFIX) #define zxc_huf_encode_section ZXC_CAT(zxc_huf_encode_section, ZXC_FUNCTION_SUFFIX) #define zxc_huf_decode_section ZXC_CAT(zxc_huf_decode_section, ZXC_FUNCTION_SUFFIX) #endif #include #include #include "../../include/zxc_error.h" #include "zxc_internal.h" /* 2048-entry multi-symbol decoder lookup table entry. Bit layout: * bits 0..7 sym1 - first decoded symbol * bits 8..15 sym2 - second decoded symbol (junk if n_extra == 0) * bits 16..19 len1 - bit length of sym1's code (1..8) * bits 20..23 len_total - total bits consumed (1..11) * bit 24 n_extra - 0 if 1 symbol, 1 if 2 symbols decoded * * Single-symbol path (tail) reads sym1 + len1; multi-symbol path (hot * batched loop) reads sym1, sym2 (always written, possibly overwritten * next iter), len_total, n_extra. */ typedef struct { uint32_t entry; } zxc_huf_dec_entry_t; #define ZXC_HUF_ENTRY(sym1, sym2, len1, len_total, n_extra) \ ((uint32_t)(sym1) | ((uint32_t)(sym2) << 8) | ((uint32_t)(len1) << 16) | \ ((uint32_t)(len_total) << 20) | ((uint32_t)(n_extra) << 24)) /* =========================================================================== * Length-limited Huffman: boundary package-merge * =========================================================================== * * Builds optimal length-limited Huffman code lengths (max length * ZXC_HUF_MAX_CODE_LEN) on 256-symbol alphabets. Package-merge is run for * ZXC_HUF_MAX_CODE_LEN levels; each level holds up to 2N items (leaves + * paired packages). Selection of the cheapest 2N - 2 items at level * ZXC_HUF_MAX_CODE_LEN gives the appearance count of each leaf, which is * its code length. */ typedef zxc_huf_pm_item_t pm_item_t; typedef struct { uint32_t w; int16_t sym; } pm_leaf_t; typedef zxc_huf_pm_frame_t frame_t; /** * @brief qsort comparator for `pm_leaf_t` arrays. * * Orders leaves by ascending weight, breaking ties by ascending symbol value * so the resulting code-length assignment is deterministic across runs. * * @param[in] a Pointer to a `pm_leaf_t` (cast from `const void*`). * @param[in] b Pointer to a `pm_leaf_t` (cast from `const void*`). * @return Negative / zero / positive per the `qsort` convention. */ static int pm_leaf_cmp(const void* a, const void* b) { const pm_leaf_t* const la = (const pm_leaf_t*)a; const pm_leaf_t* const lb = (const pm_leaf_t*)b; if (la->w < lb->w) return -1; if (la->w > lb->w) return 1; return la->sym - lb->sym; } /** * @brief Build length-limited canonical Huffman code lengths. * * Runs the boundary package-merge algorithm capped at `ZXC_HUF_MAX_CODE_LEN`. * Symbols with `freq[i] == 0` get `code_len[i] == 0`; every other symbol * receives a length in `[1, ZXC_HUF_MAX_CODE_LEN]`. The single-present-symbol * case is handled as a degenerate code of length 1. * * @param[in] freq Frequency table indexed by symbol (0..255). * @param[out] code_len Output code-length array, written in full. * @param[in] scratch Optional scratch of `ZXC_HUF_BUILD_SCRATCH_SIZE` bytes * (carved into items / counts / stack regions). If * `NULL`, the function allocates its own working memory * for the duration of the call. * @return `ZXC_OK` on success, `ZXC_ERROR_MEMORY` or `ZXC_ERROR_CORRUPT_DATA` * on failure. */ int zxc_huf_build_code_lengths(const uint32_t* RESTRICT freq, uint8_t* RESTRICT code_len, void* RESTRICT scratch) { ZXC_MEMSET(code_len, 0, ZXC_HUF_NUM_SYMBOLS); pm_leaf_t leaves[ZXC_HUF_NUM_SYMBOLS]; int n = 0; for (int i = 0; i < ZXC_HUF_NUM_SYMBOLS; i++) { if (freq[i] > 0) { leaves[n].w = freq[i]; leaves[n].sym = (int16_t)i; n++; } } if (UNLIKELY(n == 0)) return ZXC_ERROR_CORRUPT_DATA; if (n == 1) { code_len[leaves[0].sym] = 1; return ZXC_OK; } qsort(leaves, (size_t)n, sizeof(pm_leaf_t), pm_leaf_cmp); /* n <= 256 <= 2^ZXC_HUF_MAX_CODE_LEN, so length-limit is always feasible. */ const int max_per_level = 2 * n; /* Working buffers: either carve from caller-provided scratch (sized for * the worst-case alphabet) or fall back to per-call malloc/free. */ pm_item_t* items; int* counts; frame_t* stack; pm_item_t* owned_items = NULL; int* owned_counts = NULL; frame_t* owned_stack = NULL; if (scratch) { uint8_t* p = (uint8_t*)scratch; items = (pm_item_t*)p; p += (size_t)ZXC_HUF_MAX_CODE_LEN * (size_t)ZXC_HUF_PM_LEVEL_BOUND * sizeof(pm_item_t); p = (uint8_t*)(((uintptr_t)p + 7u) & ~(uintptr_t)7u); counts = (int*)p; ZXC_MEMSET(counts, 0, (size_t)ZXC_HUF_MAX_CODE_LEN * sizeof(int)); p += (size_t)ZXC_HUF_MAX_CODE_LEN * sizeof(int); p = (uint8_t*)(((uintptr_t)p + 7u) & ~(uintptr_t)7u); stack = (frame_t*)p; } else { owned_items = (pm_item_t*)malloc((size_t)ZXC_HUF_MAX_CODE_LEN * (size_t)max_per_level * sizeof(pm_item_t)); owned_counts = (int*)calloc((size_t)ZXC_HUF_MAX_CODE_LEN, sizeof(int)); owned_stack = (frame_t*)malloc((size_t)ZXC_HUF_MAX_CODE_LEN * (size_t)max_per_level * sizeof(frame_t)); if (UNLIKELY(!owned_items || !owned_counts || !owned_stack)) { free(owned_items); free(owned_counts); free(owned_stack); return ZXC_ERROR_MEMORY; } items = owned_items; counts = owned_counts; stack = owned_stack; } #define ITEM(k, i) items[(size_t)(k) * (size_t)max_per_level + (size_t)(i)] /* Level 0 (logical level 1): the leaves themselves, already sorted. */ for (int i = 0; i < n; i++) { ITEM(0, i).weight = leaves[i].w; ITEM(0, i).left = -1; ITEM(0, i).right = -1; ITEM(0, i).sym = leaves[i].sym; } counts[0] = n; /* Levels 1..ZXC_HUF_MAX_CODE_LEN-1: merge sorted leaves with sorted packages from the previous * level. */ for (int k = 1; k < ZXC_HUF_MAX_CODE_LEN; k++) { const int prev = counts[k - 1]; const int packs = prev / 2; int li = 0, pi = 0, n_lvl = 0; while (li < n || pi < packs) { const uint32_t wl = (li < n) ? leaves[li].w : UINT32_MAX; const uint32_t wp = (pi < packs) ? (uint32_t)(ITEM(k - 1, 2 * pi).weight + ITEM(k - 1, 2 * pi + 1).weight) : UINT32_MAX; if (wl <= wp && li < n) { ITEM(k, n_lvl).weight = wl; ITEM(k, n_lvl).left = -1; ITEM(k, n_lvl).right = -1; ITEM(k, n_lvl).sym = leaves[li].sym; li++; } else { ITEM(k, n_lvl).weight = wp; ITEM(k, n_lvl).left = (int16_t)(2 * pi); ITEM(k, n_lvl).right = (int16_t)(2 * pi + 1); ITEM(k, n_lvl).sym = -1; pi++; } n_lvl++; } counts[k] = n_lvl; } /* Step 3: take first 2n-2 items at level ZXC_HUF_MAX_CODE_LEN-1; trace back, counting leaf * appearances. */ int n_take = 2 * n - 2; if (n_take > counts[ZXC_HUF_MAX_CODE_LEN - 1]) n_take = counts[ZXC_HUF_MAX_CODE_LEN - 1]; /* Worst case stack depth: (ZXC_HUF_MAX_CODE_LEN * n_take) frames; bounded by * ZXC_HUF_MAX_CODE_LEN * 2n. `stack` was set up earlier from scratch (or * the local malloc fallback). */ int sp = 0; for (int i = 0; i < n_take; i++) { stack[sp].lvl = (int8_t)(ZXC_HUF_MAX_CODE_LEN - 1); stack[sp].idx = (int16_t)i; sp++; } while (sp > 0) { frame_t f = stack[--sp]; const pm_item_t* it = &ITEM(f.lvl, f.idx); if (it->sym >= 0) { code_len[it->sym]++; } else { stack[sp].lvl = (int8_t)(f.lvl - 1); stack[sp].idx = it->left; sp++; stack[sp].lvl = (int8_t)(f.lvl - 1); stack[sp].idx = it->right; sp++; } } if (owned_items) { free(owned_items); free(owned_counts); free(owned_stack); } #undef ITEM return ZXC_OK; } /* =========================================================================== * Canonical code construction (LSB-first by bit-reversing canonical MSB codes) * =========================================================================*/ /** * @brief Reverse the low @p n bits of @p v. * * Used to convert MSB-first canonical Huffman codes (the natural form * produced by the canonical-code construction) into LSB-first codes that * can be packed into the bit writer with a single shift-or. * * @param[in] v Value whose low @p n bits will be reversed. * @param[in] n Number of significant bits in @p v (1..32). * @return The bit-reversed value, with bits above position @p n set to 0. */ static uint32_t reverse_bits(uint32_t v, const int n) { uint32_t r = 0; for (int i = 0; i < n; i++) { r = (r << 1) | (v & 1u); v >>= 1; } return r; } /** * @brief Build the canonical LSB-first Huffman codes for a length table. * * Generates MSB-first canonical codes following RFC 1951 3.2.2, then * bit-reverses each so the encoder can emit them with a plain * `accum |= code << bits` step. Absent symbols (length 0) receive code 0. * * @param[in] code_len Per-symbol code lengths. * @param[out] codes Per-symbol LSB-first canonical codes. */ static void build_canonical_codes(const uint8_t* RESTRICT code_len, uint32_t* RESTRICT codes) { uint32_t bl_count[ZXC_HUF_MAX_CODE_LEN + 1] = {0}; for (int i = 0; i < ZXC_HUF_NUM_SYMBOLS; i++) { bl_count[code_len[i]]++; } bl_count[0] = 0; uint32_t next_code[ZXC_HUF_MAX_CODE_LEN + 2] = {0}; uint32_t code = 0; for (int k = 1; k <= ZXC_HUF_MAX_CODE_LEN + 1; k++) { code = (code + bl_count[k - 1]) << 1; next_code[k] = code; } for (int i = 0; i < ZXC_HUF_NUM_SYMBOLS; i++) { const int l = code_len[i]; if (l == 0) { codes[i] = 0; } else { const uint32_t msb_code = next_code[l]++; codes[i] = reverse_bits(msb_code, l); } } } /* =========================================================================== * 128-byte length header: 256 x 4-bit lengths, low nibble first. * =========================================================================*/ /** * @brief Pack 256 4-bit code lengths into the 128-byte section header. * * The packing is little-endian within each byte: low nibble holds * `code_len[2*i]`, high nibble holds `code_len[2*i + 1]`. The function * silently truncates any length > 15; callers must enforce the cap of * `ZXC_HUF_MAX_CODE_LEN` (<= 15) before calling. * * @param[in] code_len Per-symbol code lengths (length `ZXC_HUF_NUM_SYMBOLS`). * @param[out] out Output header buffer of `ZXC_HUF_LENGTHS_HEADER_SIZE` bytes. */ static void pack_lengths_header(const uint8_t* RESTRICT code_len, uint8_t* RESTRICT out) { for (int i = 0; i < ZXC_HUF_NUM_SYMBOLS; i += 2) { const uint8_t lo = code_len[i] & 0x0F; const uint8_t hi = code_len[i + 1] & 0x0F; out[i >> 1] = (uint8_t)(lo | (hi << 4)); } } /** * @brief Decode the 128-byte length header back into 256 code lengths. * * Inverts ::pack_lengths_header and validates the two structural invariants: * no length exceeds `ZXC_HUF_MAX_CODE_LEN`, and at least one symbol is * present. * * @param[in] in Input header buffer of `ZXC_HUF_LENGTHS_HEADER_SIZE` bytes. * @param[out] code_len Output code-length array of length `ZXC_HUF_NUM_SYMBOLS`. * @return `ZXC_OK` on success, `ZXC_ERROR_CORRUPT_DATA` if a length is too * large or the table is empty. */ static int unpack_lengths_header(const uint8_t* RESTRICT in, uint8_t* RESTRICT code_len) { int max_len = 0; int n_present = 0; for (int i = 0; i < ZXC_HUF_NUM_SYMBOLS; i += 2) { const uint8_t b = in[i >> 1]; const uint8_t lo = b & 0x0F; const uint8_t hi = (uint8_t)(b >> 4); code_len[i] = lo; code_len[i + 1] = hi; if (lo > max_len) max_len = lo; if (hi > max_len) max_len = hi; if (lo) n_present++; if (hi) n_present++; } if (UNLIKELY(max_len > ZXC_HUF_MAX_CODE_LEN)) return ZXC_ERROR_CORRUPT_DATA; if (UNLIKELY(n_present == 0)) return ZXC_ERROR_CORRUPT_DATA; return ZXC_OK; } /* =========================================================================== * Bit writer (LSB-first) * =========================================================================*/ typedef struct { uint8_t* ptr; uint8_t* end; uint64_t accum; int bits; int err; } bit_writer_t; /** * @brief Initialise an LSB-first bit writer over a caller-owned buffer. * * @param[out] bw Writer to initialise. * @param[out] dst Output buffer (writer takes no ownership). * @param[in] cap Capacity of @p dst in bytes. */ static ZXC_ALWAYS_INLINE void bw_init(bit_writer_t* RESTRICT bw, uint8_t* RESTRICT dst, const size_t cap) { bw->ptr = dst; bw->end = dst + cap; bw->accum = 0; bw->bits = 0; bw->err = 0; } /** * @brief Append the low @p len bits of @p code to the writer's bitstream. * * Bits are consumed from the LSB end. When the internal accumulator has * accumulated 8 or more bits, full bytes are flushed to the output buffer. * If the buffer is exhausted mid-flush the writer's `err` flag is set; * subsequent ::bw_finish reports `ZXC_ERROR_DST_TOO_SMALL`. * * @param[in,out] bw Writer state. * @param[in] code Code bits to emit (the low @p len bits matter). * @param[in] len Number of bits to emit (1..ZXC_HUF_MAX_CODE_LEN). */ static ZXC_ALWAYS_INLINE void bw_put(bit_writer_t* RESTRICT bw, const uint32_t code, const int len) { bw->accum |= ((uint64_t)code) << bw->bits; bw->bits += len; if (LIKELY((size_t)(bw->end - bw->ptr) >= sizeof(uint64_t))) { zxc_store_le64(bw->ptr, bw->accum); } else { if (UNLIKELY(bw->ptr >= bw->end)) { bw->err = 1; bw->bits = 0; return; } *bw->ptr = (uint8_t)bw->accum; } const int n = bw->bits >> 3; /* 0 or 1 full byte to flush */ bw->ptr += n; bw->accum >>= n << 3; bw->bits &= 7; } /** * @brief Flush any partial trailing byte and finalise the bit writer. * * Writes the (zero-padded) trailing byte if the accumulator holds any bits. * * @param[in,out] bw Writer state. * @return `ZXC_OK` on success, `ZXC_ERROR_DST_TOO_SMALL` if the buffer was * exhausted at any point. */ static ZXC_ALWAYS_INLINE int bw_finish(bit_writer_t* RESTRICT bw) { if (bw->bits > 0) { if (UNLIKELY(bw->ptr >= bw->end)) return ZXC_ERROR_DST_TOO_SMALL; *bw->ptr++ = (uint8_t)bw->accum; bw->accum = 0; bw->bits = 0; } return UNLIKELY(bw->err) ? ZXC_ERROR_DST_TOO_SMALL : ZXC_OK; } /* =========================================================================== * Encoder * =========================================================================*/ /** * @copydoc zxc_huf_encode_section */ int zxc_huf_encode_section(const uint8_t* RESTRICT literals, const size_t n_literals, const uint8_t* RESTRICT code_len, uint8_t* RESTRICT dst, const size_t dst_cap) { if (UNLIKELY(n_literals == 0)) return ZXC_ERROR_CORRUPT_DATA; if (UNLIKELY(dst_cap < ZXC_HUF_HEADER_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; /* 1. Pack the 128-byte length header. */ pack_lengths_header(code_len, dst); /* 2. Build canonical codes (LSB-first via bit-reversal). */ uint32_t codes[ZXC_HUF_NUM_SYMBOLS]; build_canonical_codes(code_len, codes); /* 3. Reserve 6 bytes for sub-stream sizes; encode 4 sub-streams after them. */ uint8_t* const sizes_hdr = dst + ZXC_HUF_LENGTHS_HEADER_SIZE; uint8_t* const stream_base = sizes_hdr + ZXC_HUF_STREAM_SIZES_HEADER_SIZE; const uint8_t* const stream_end = dst + dst_cap; const size_t Q = (n_literals + ZXC_HUF_NUM_STREAMS - 1) / ZXC_HUF_NUM_STREAMS; bit_writer_t bw; uint8_t* p = stream_base; size_t s_sizes[ZXC_HUF_NUM_STREAMS]; for (int s = 0; s < ZXC_HUF_NUM_STREAMS; s++) { const size_t start = (size_t)s * Q; size_t stop = start + Q; if (stop > n_literals) stop = n_literals; uint8_t* const stream_start = p; bw_init(&bw, p, (size_t)(stream_end - p)); for (size_t i = start; i < stop; i++) { const uint8_t sym = literals[i]; const int len = code_len[sym]; if (UNLIKELY(len == 0)) return ZXC_ERROR_CORRUPT_DATA; /* symbol absent from table */ bw_put(&bw, codes[sym], len); } const int rc = bw_finish(&bw); if (UNLIKELY(rc != ZXC_OK)) return rc; s_sizes[s] = (size_t)(bw.ptr - stream_start); p = bw.ptr; } /* 4. Persist the 3 explicit sub-stream sizes (s4 is implied). */ for (int s = 0; s < ZXC_HUF_NUM_STREAMS - 1; s++) { if (UNLIKELY(s_sizes[s] > 0xFFFFu)) return ZXC_ERROR_DST_TOO_SMALL; zxc_store_le16(sizes_hdr + 2 * s, (uint16_t)s_sizes[s]); } return (int)(p - dst); } /* =========================================================================== * Decoder table builder + 4-way interleaved decoder * =========================================================================*/ /** * @brief Build the 2048-entry multi-symbol decoder lookup table. * * Strategy: build a temporary 256-entry single-symbol (8-bit) table, then * use it to populate the 2048-entry (11-bit) multi-symbol table. For each * 11-bit prefix p: * 1. (sym1, len1) = ss[p & 0xFF] -- always valid, 1 <= len1 <= 8. * 2. rem = 11 - len1 E [3, 10] bits remain after consuming the first code. * 3. (sym2_cand, len2_cand) = ss[(p >> len1) & 0xFF]. If len2_cand <= rem, * both codes fit in 11 bits -> encode 2-symbol entry. Otherwise the * second code's bit window extends past the lookup width -> keep only * the first symbol and let the next iteration handle the rest. * * Validates Kraft equality (or the single-present-symbol degenerate case). * * @param[in] code_len Per-symbol code lengths from the section header. * @param[out] table Destination 2048-entry lookup table (caller-aligned). * @return `ZXC_OK` on success, `ZXC_ERROR_CORRUPT_DATA` on validation failure. */ static int build_decode_table(const uint8_t* RESTRICT code_len, zxc_huf_dec_entry_t* RESTRICT table) { uint32_t bl_count[ZXC_HUF_MAX_CODE_LEN + 1] = {0}; int n_present = 0; for (int i = 0; i < ZXC_HUF_NUM_SYMBOLS; i++) { const uint8_t l = code_len[i]; if (UNLIKELY(l > ZXC_HUF_MAX_CODE_LEN)) return ZXC_ERROR_CORRUPT_DATA; bl_count[l]++; if (l) n_present++; } if (UNLIKELY(n_present == 0)) return ZXC_ERROR_CORRUPT_DATA; bl_count[0] = 0; /* Validate Kraft equality on the ZXC_HUF_MAX_CODE_LEN axis. */ { uint64_t kraft = 0; for (int k = 1; k <= ZXC_HUF_MAX_CODE_LEN; k++) { kraft += (uint64_t)bl_count[k] << (ZXC_HUF_MAX_CODE_LEN - k); } /* Degenerate: single symbol with length 1 (Kraft sum = * 2^(ZXC_HUF_MAX_CODE_LEN-1)). Otherwise: full Kraft equality * on the ZXC_HUF_MAX_CODE_LEN axis. */ const int kraft_ok = (n_present == 1) ? (bl_count[1] == 1) : (kraft == ((uint64_t)1 << ZXC_HUF_MAX_CODE_LEN)); if (UNLIKELY(!kraft_ok)) return ZXC_ERROR_CORRUPT_DATA; } uint32_t next_code[ZXC_HUF_MAX_CODE_LEN + 2] = {0}; { uint32_t code = 0; for (int k = 1; k <= ZXC_HUF_MAX_CODE_LEN + 1; k++) { code = (code + bl_count[k - 1]) << 1; next_code[k] = code; } } /* Single-symbol intermediate (ZXC_HUF_MAX_CODE_LEN-bit lookup). Layout: * low byte = sym, high byte = len. Filled by replicating each canonical * code across all ZXC_HUF_MAX_CODE_LEN-bit windows that share its low * `len` bits. */ #define ZXC_HUF_SS_SIZE (1u << ZXC_HUF_MAX_CODE_LEN) #define ZXC_HUF_SS_MASK ((uint32_t)(ZXC_HUF_SS_SIZE - 1)) uint16_t ss[ZXC_HUF_SS_SIZE] = {0}; for (int sym = 0; sym < ZXC_HUF_NUM_SYMBOLS; sym++) { const int l = code_len[sym]; if (l == 0) continue; const uint32_t msb_code = next_code[l]++; const uint32_t lsb_code = reverse_bits(msb_code, l); const uint16_t entry = (uint16_t)((unsigned)l << 8 | (unsigned)sym); const uint32_t step = (uint32_t)1 << l; for (uint32_t fill = lsb_code; fill < ZXC_HUF_SS_SIZE; fill += step) { ss[fill] = entry; } } /* Single-symbol degenerate (Kraft sum = 2^(ZXC_HUF_MAX_CODE_LEN-1)): replicate the one * valid entry across every slot. */ if (UNLIKELY(n_present == 1)) { uint16_t valid = 0; for (uint32_t i = 0; i < ZXC_HUF_SS_SIZE; i++) { if (ss[i] != 0) { valid = ss[i]; break; } } for (uint32_t i = 0; i < ZXC_HUF_SS_SIZE; i++) { if (ss[i] == 0) ss[i] = valid; } } /* Build the multi-symbol table. */ for (uint32_t p = 0; p < ZXC_HUF_TABLE_SIZE; p++) { const uint16_t e1 = ss[p & ZXC_HUF_SS_MASK]; const uint8_t sym1 = (uint8_t)e1; const int len1 = e1 >> 8; const int rem = ZXC_HUF_LOOKUP_BITS - len1; uint8_t sym2 = 0; int len_total = len1; int n_extra = 0; const uint16_t e2 = ss[(p >> len1) & ZXC_HUF_SS_MASK]; const int len2 = e2 >> 8; if (len2 <= rem) { sym2 = (uint8_t)e2; len_total = len1 + len2; n_extra = 1; } table[p].entry = ZXC_HUF_ENTRY(sym1, sym2, len1, len_total, n_extra); } #undef ZXC_HUF_SS_SIZE #undef ZXC_HUF_SS_MASK return ZXC_OK; } /** * @copydoc zxc_huf_decode_section */ int zxc_huf_decode_section(const uint8_t* RESTRICT payload, const size_t payload_size, uint8_t* RESTRICT dst, const size_t n_literals) { if (UNLIKELY(payload_size < ZXC_HUF_HEADER_SIZE || n_literals == 0)) return ZXC_ERROR_CORRUPT_DATA; /* 1. Parse length header. */ uint8_t code_len[ZXC_HUF_NUM_SYMBOLS]; { const int rc = unpack_lengths_header(payload, code_len); if (UNLIKELY(rc != ZXC_OK)) return rc; } /* 2. Build the 2048-entry multi-symbol decode table. Cache-line * aligned: the LUT spans 128 lines (8 KB / 64 B) and is hammered every * symbol, landing it on a 64-byte boundary avoids any cross-line * load split on the per-iteration entry fetch. */ ZXC_ALIGN(ZXC_CACHE_LINE_SIZE) zxc_huf_dec_entry_t table[ZXC_HUF_TABLE_SIZE]; { const int rc = build_decode_table(code_len, table); if (UNLIKELY(rc != ZXC_OK)) return rc; } /* 3. Parse sub-stream sizes. */ const uint8_t* const sizes_hdr = payload + ZXC_HUF_LENGTHS_HEADER_SIZE; const uint16_t s1 = zxc_le16(sizes_hdr + 0); const uint16_t s2 = zxc_le16(sizes_hdr + 2); const uint16_t s3 = zxc_le16(sizes_hdr + 4); const size_t streams_total = payload_size - ZXC_HUF_HEADER_SIZE; const size_t s123 = (size_t)s1 + (size_t)s2 + (size_t)s3; if (UNLIKELY(s123 > streams_total)) return ZXC_ERROR_CORRUPT_DATA; const size_t s4 = streams_total - s123; const uint8_t* const stream_base = payload + ZXC_HUF_HEADER_SIZE; const size_t off[ZXC_HUF_NUM_STREAMS] = {0, s1, (size_t)s1 + s2, s123}; const size_t sz[ZXC_HUF_NUM_STREAMS] = {s1, s2, s3, s4}; /* 4. Initialise 4 bit readers. */ zxc_bit_reader_t br[ZXC_HUF_NUM_STREAMS]; for (int s = 0; s < ZXC_HUF_NUM_STREAMS; s++) { zxc_br_init(&br[s], stream_base + off[s], sz[s]); } /* 5. 4-way interleaved multi-symbol decode. Each sub-stream owns a * contiguous slice of dst: stream s covers literal indices * [s*Q, min((s+1)*Q, N)). With Q = ceil(N/4) the first 3 streams have * exactly Q symbols and stream 3 has `N - 3Q` symbols. */ const size_t Q = (n_literals + ZXC_HUF_NUM_STREAMS - 1) / ZXC_HUF_NUM_STREAMS; size_t s_count[ZXC_HUF_NUM_STREAMS]; uint8_t* s_dst[ZXC_HUF_NUM_STREAMS]; for (int s = 0; s < ZXC_HUF_NUM_STREAMS; s++) { size_t start = (size_t)s * Q; size_t stop = start + Q; if (start > n_literals) start = n_literals; if (stop > n_literals) stop = n_literals; s_count[s] = stop - start; s_dst[s] = dst + start; } /* Batched multi-symbol decode. Each ZXC_HUF_BATCH iterations consume * <= ZXC_HUF_BATCH_BITS bits per stream, fitting under the 57-bit cap * an 8-byte refill can guarantee. * * Each iter speculatively writes 2 bytes per stream and advances by 1 * or 2. If only 1 symbol was decoded, the spec byte is overwritten by * the next iter, except at end-of-stream where it would corrupt the * adjacent stream. The batched loop therefore requires * ZXC_HUF_SAFE_MARGIN bytes of headroom per stream. */ uint8_t* d0 = s_dst[0]; uint8_t* d1 = s_dst[1]; uint8_t* d2 = s_dst[2]; uint8_t* d3 = s_dst[3]; const uint8_t* const dend0 = s_dst[0] + s_count[0]; const uint8_t* const dend1 = s_dst[1] + s_count[1]; const uint8_t* const dend2 = s_dst[2] + s_count[2]; const uint8_t* const dend3 = s_dst[3] + s_count[3]; /* Hoist all four bit-reader hot fields into locals. They live in * registers for the full duration of the batched loop. */ uint64_t a0 = br[0].accum, a1 = br[1].accum, a2 = br[2].accum, a3 = br[3].accum; int bb0 = br[0].bits, bb1 = br[1].bits, bb2 = br[2].bits, bb3 = br[3].bits; const uint8_t *p0 = br[0].ptr, *p1 = br[1].ptr, *p2 = br[2].ptr, *p3 = br[3].ptr; const uint8_t* const e0 = br[0].end; const uint8_t* const e1 = br[1].end; const uint8_t* const e2 = br[2].end; const uint8_t* const e3 = br[3].end; /* Refill the bit accumulator with up to (ZXC_HUF_ACCUM_BITS - nbits) more * bits read from src. Fast path reads 8 bytes at once (LE u64 load); slow * path reads byte-by-byte while at least one byte of free room remains. */ #define REFILL(accum, nbits, src, src_end) \ do { \ if (LIKELY((nbits) < ZXC_HUF_BATCH_BITS && (src) + sizeof(uint64_t) <= (src_end))) { \ (accum) |= zxc_le64(src) << (nbits); \ const int _n = (ZXC_HUF_ACCUM_BITS - (nbits)) / CHAR_BIT; \ (src) += _n; \ (nbits) += _n * CHAR_BIT; \ } else { \ while ((nbits) <= ZXC_HUF_ACCUM_BITS - CHAR_BIT && (src) < (src_end)) { \ (accum) |= ((uint64_t)*(src)++) << (nbits); \ (nbits) += CHAR_BIT; \ } \ } \ } while (0) /* Decode one 11-bit window per stream. Always writes 2 bytes per stream * (sym1 + spec sym2); advances d_s by 1 + n_extra; advances accum by * len_total. Per-stream length accumulators sl0..sl3 collect consumed * bits across the batch and are folded into bb_s once at end of batch. */ #define DECODE_ONE() \ do { \ const uint32_t _e0 = table[a0 & ZXC_HUF_TBL_MASK].entry; \ const uint32_t _e1 = table[a1 & ZXC_HUF_TBL_MASK].entry; \ const uint32_t _e2 = table[a2 & ZXC_HUF_TBL_MASK].entry; \ const uint32_t _e3 = table[a3 & ZXC_HUF_TBL_MASK].entry; \ zxc_store_le16(d0, (uint16_t)_e0); \ zxc_store_le16(d1, (uint16_t)_e1); \ zxc_store_le16(d2, (uint16_t)_e2); \ zxc_store_le16(d3, (uint16_t)_e3); \ const int _t0 = (int)((_e0 >> 20) & 0xF); \ const int _t1 = (int)((_e1 >> 20) & 0xF); \ const int _t2 = (int)((_e2 >> 20) & 0xF); \ const int _t3 = (int)((_e3 >> 20) & 0xF); \ d0 += 1 + (int)((_e0 >> 24) & 1); \ d1 += 1 + (int)((_e1 >> 24) & 1); \ d2 += 1 + (int)((_e2 >> 24) & 1); \ d3 += 1 + (int)((_e3 >> 24) & 1); \ a0 >>= _t0; \ a1 >>= _t1; \ a2 >>= _t2; \ a3 >>= _t3; \ sl0 += _t0; \ sl1 += _t1; \ sl2 += _t2; \ sl3 += _t3; \ } while (0) while ((size_t)(dend0 - d0) >= ZXC_HUF_SAFE_MARGIN && (size_t)(dend1 - d1) >= ZXC_HUF_SAFE_MARGIN && (size_t)(dend2 - d2) >= ZXC_HUF_SAFE_MARGIN && (size_t)(dend3 - d3) >= ZXC_HUF_SAFE_MARGIN) { REFILL(a0, bb0, p0, e0); REFILL(a1, bb1, p1, e1); REFILL(a2, bb2, p2, e2); REFILL(a3, bb3, p3, e3); int sl0 = 0, sl1 = 0, sl2 = 0, sl3 = 0; DECODE_ONE(); DECODE_ONE(); DECODE_ONE(); DECODE_ONE(); DECODE_ONE(); bb0 -= sl0; bb1 -= sl1; bb2 -= sl2; bb3 -= sl3; } /* Per-stream scalar tail (<= ZXC_HUF_SAFE_MARGIN - 1 = 9 symbols per * stream). Single-symbol decode using the same 2048-entry table, * we read sym1 + len1 only and advance by 1 byte, no spec write. */ #define TAIL_ONE(accum, nbits, src, src_end, dst) \ do { \ REFILL(accum, nbits, src, src_end); \ const uint32_t _e = table[(accum) & ZXC_HUF_TBL_MASK].entry; \ *(dst)++ = (uint8_t)_e; \ const int _l1 = (int)((_e >> 16) & 0xF); \ (accum) >>= _l1; \ (nbits) -= _l1; \ } while (0) while (d0 < dend0) TAIL_ONE(a0, bb0, p0, e0, d0); while (d1 < dend1) TAIL_ONE(a1, bb1, p1, e1, d1); while (d2 < dend2) TAIL_ONE(a2, bb2, p2, e2, d2); while (d3 < dend3) TAIL_ONE(a3, bb3, p3, e3, d3); #undef TAIL_ONE #undef DECODE_ONE #undef REFILL return ZXC_OK; } zxc-0.11.0/src/lib/zxc_internal.h000066400000000000000000001671361520102567100166200ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /** * @file zxc_internal.h * @brief Internal definitions, constants, SIMD helpers, and utility functions. * * This header is **not** part of the public API. It is shared across the * library's translation units and contains: * - Platform detection and SIMD intrinsic includes. * - Compiler-abstraction macros (LIKELY, PREFETCH, MEMCPY, ALIGN, ...). * - Endianness detection and byte-swap helpers. * - File-format constants (magic word, header sizes, block sizes, ...). * - Inline helpers for hashing, endian-safe loads/stores, bit manipulation, * ZigZag encoding, aligned allocation, and bitstream reading. * - Internal function prototypes for chunk-level compression/decompression. * * @warning Do not include this header from user code; use the public headers * zxc_buffer.h, zxc_stream.h, or zxc_sans_io.h instead. */ #ifndef ZXC_INTERNAL_H #define ZXC_INTERNAL_H #include #include #include #include #include #include #include #include "../../include/zxc_buffer.h" #include "../../include/zxc_constants.h" #include "../../include/zxc_sans_io.h" #include "rapidhash.h" #ifdef __cplusplus extern "C" { #endif /** * @defgroup internal Internal Helpers * @brief Platform abstractions, constants, and utility functions (private). * @{ */ /** * @name Atomic Qualifier * @brief Provides a portable atomic / volatile qualifier. * * If C11 atomics are available, @c ZXC_ATOMIC expands to @c _Atomic; * otherwise it falls back to @c volatile. * @{ */ #if !defined(__cplusplus) && defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L && \ !defined(__STDC_NO_ATOMICS__) #include #define ZXC_ATOMIC _Atomic #define ZXC_USE_C11_ATOMICS 1 #else #define ZXC_ATOMIC volatile #define ZXC_USE_C11_ATOMICS 0 #endif /** @} */ /* end of Atomic Qualifier */ /** * @name SIMD Intrinsics & Compiler Macros * @brief Auto-detected SIMD feature macros for x86 (SSE/AVX) and ARM (NEON). * * Depending on the target architecture and compiler flags the following macros * may be defined: * - @c ZXC_USE_AVX512 - AVX-512F + AVX-512BW available. * - @c ZXC_USE_AVX2 - AVX2 available. * - @c ZXC_USE_NEON64 - AArch64 NEON available. * - @c ZXC_USE_NEON32 - ARMv7 NEON available. * * Define @c ZXC_DISABLE_SIMD to gate all hand-written SIMD paths (intrinsics, * inline assembly). Compiler auto-vectorisation is unaffected. * @{ */ #ifndef ZXC_DISABLE_SIMD #if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) #include #include #if defined(__AVX512F__) && defined(__AVX512BW__) #ifndef ZXC_USE_AVX512 #define ZXC_USE_AVX512 #endif #endif #if defined(__AVX2__) #ifndef ZXC_USE_AVX2 #define ZXC_USE_AVX2 #endif #endif #elif (defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(_M_ARM64) || \ defined(ZXC_USE_NEON32) || defined(ZXC_USE_NEON64)) #if !defined(_MSC_VER) #include #endif #include #if defined(__aarch64__) || defined(_M_ARM64) #ifndef ZXC_USE_NEON64 #define ZXC_USE_NEON64 #endif #else #ifndef ZXC_USE_NEON32 #define ZXC_USE_NEON32 #endif #endif #endif #endif /* ZXC_DISABLE_SIMD */ /** @} */ /* end of SIMD Intrinsics */ /** * @name Compiler Abstractions * @brief Portable wrappers for branch hints, prefetch, memory ops, alignment, * and forced inlining. * @{ */ #if defined(__GNUC__) || defined(__clang__) /** @def LIKELY * @brief Branch prediction hint: expression is likely true. * @param x Expression to evaluate. */ #define LIKELY(x) (__builtin_expect(!!(x), 1)) /** @def UNLIKELY * @brief Branch prediction hint: expression is unlikely to be true. * @param x Expression to evaluate. */ #define UNLIKELY(x) (__builtin_expect(!!(x), 0)) /** @def RESTRICT * @brief Pointer aliasing hint (maps to __restrict__). */ #define RESTRICT __restrict__ /** @def ZXC_PREFETCH_READ * @brief Prefetch data for reading. * @param ptr Pointer to data to prefetch. */ #define ZXC_PREFETCH_READ(ptr) __builtin_prefetch((const void*)(ptr), 0, 3) /** @def ZXC_PREFETCH_WRITE * @brief Prefetch data for writing. * @param ptr Pointer to data to prefetch. */ #define ZXC_PREFETCH_WRITE(ptr) __builtin_prefetch((const void*)(ptr), 1, 3) /** @def ZXC_MEMCPY * @brief Optimized memory copy using compiler built-in. */ #define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) /** @def ZXC_MEMSET * @brief Optimized memory set using compiler built-in. */ #define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) /** @def ZXC_ALIGN * @brief Specifies memory alignment for a variable or structure. * @param x Alignment boundary in bytes (must be a power of 2). */ #define ZXC_ALIGN(x) __attribute__((aligned(x))) /** @def ZXC_ALWAYS_INLINE * @brief Forces a function to be inlined at all optimization levels. */ #define ZXC_ALWAYS_INLINE inline __attribute__((always_inline)) #elif defined(_MSC_VER) #include #if defined(_M_IX86) || defined(_M_X64) || defined(_M_AMD64) #include #define ZXC_PREFETCH_READ(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) #define ZXC_PREFETCH_WRITE(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) #else #define ZXC_PREFETCH_READ(ptr) __prefetch((const void*)(ptr)) #define ZXC_PREFETCH_WRITE(ptr) __prefetch((const void*)(ptr)) #endif #define LIKELY(x) (x) #define UNLIKELY(x) (x) #define RESTRICT __restrict #pragma intrinsic(memcpy, memset) #define ZXC_MEMCPY(dst, src, n) memcpy(dst, src, n) #define ZXC_MEMSET(dst, val, n) memset(dst, val, n) /** @def ZXC_ALIGN * @brief Specifies memory alignment for a variable or structure (MSVC). * @param x Alignment boundary in bytes (must be a power of 2). */ #define ZXC_ALIGN(x) __declspec(align(x)) /** @def ZXC_ALWAYS_INLINE * @brief Forces a function to be inlined at all optimization levels (MSVC). */ #define ZXC_ALWAYS_INLINE __forceinline #pragma intrinsic(_BitScanReverse) #else #define LIKELY(x) (x) #define UNLIKELY(x) (x) #define RESTRICT #define ZXC_PREFETCH_READ(ptr) #define ZXC_PREFETCH_WRITE(ptr) #define ZXC_MEMCPY(dst, src, n) memcpy(dst, src, n) #define ZXC_MEMSET(dst, val, n) memset(dst, val, n) /** @def ZXC_ALWAYS_INLINE * @brief Forces a function to be inlined (fallback for non-GCC/Clang/MSVC compilers). */ #define ZXC_ALWAYS_INLINE inline #if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L #include /** @def ZXC_ALIGN * @brief Specifies memory alignment using C11 _Alignas. * @param x Alignment boundary in bytes (must be a power of 2). */ #define ZXC_ALIGN(x) _Alignas(x) #else /** @def ZXC_ALIGN * @brief No-op alignment macro for compilers without alignment support. * @param x Ignored (alignment not supported). */ #define ZXC_ALIGN(x) #endif #endif /** @} */ /* end of Compiler Abstractions */ /** * @name Endianness Detection * @brief Compile-time detection of host byte order. * * Defines exactly one of @c ZXC_LITTLE_ENDIAN or @c ZXC_BIG_ENDIAN. * @{ */ #ifndef ZXC_LITTLE_ENDIAN #if defined(_WIN32) || defined(__LITTLE_ENDIAN__) || \ (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) #define ZXC_LITTLE_ENDIAN #elif defined(__BIG_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) #define ZXC_BIG_ENDIAN #else #warning "Endianness not detected, defaulting to little-endian" #define ZXC_LITTLE_ENDIAN #endif #endif /** @} */ /* end of Endianness Detection */ /** * @name Byte-Swap Helpers * @brief 16/32/64-bit byte-swap macros (only defined under @c ZXC_BIG_ENDIAN). * @{ */ #ifdef ZXC_BIG_ENDIAN #if defined(__GNUC__) || defined(__clang__) #define ZXC_BSWAP16(x) __builtin_bswap16(x) #define ZXC_BSWAP32(x) __builtin_bswap32(x) #define ZXC_BSWAP64(x) __builtin_bswap64(x) #elif defined(_MSC_VER) #define ZXC_BSWAP16(x) _byteswap_ushort(x) #define ZXC_BSWAP32(x) _byteswap_ulong(x) #define ZXC_BSWAP64(x) _byteswap_uint64(x) #else #define ZXC_BSWAP16(x) ((uint16_t)(((x) >> 8) | ((x) << 8))) #define ZXC_BSWAP32(x) \ ((uint32_t)(((x) >> 24) | (((x) >> 8) & 0xFF00) | (((x) << 8) & 0xFF0000) | ((x) << 24))) #define ZXC_BSWAP64(x) \ ((uint64_t)(((uint64_t)ZXC_BSWAP32((uint32_t)(x)) << 32) | ZXC_BSWAP32((uint32_t)((x) >> 32)))) #endif #endif /** @} */ /* end of Byte-Swap Helpers */ /** * @name File Format Constants * @brief Magic words, header sizes, block sizes, and related constants. * @{ */ /** @brief Magic word identifying ZXC files (little-endian 0x9CB02EF5). */ #define ZXC_MAGIC_WORD 0x9CB02EF5U /** @brief Current on-disk file format version. */ #define ZXC_FILE_FORMAT_VERSION 5 /** @brief Size of stdio I/O buffers (1 MB). */ #define ZXC_IO_BUFFER_SIZE (1024 * 1024) /** @brief Maximum number of threads allowed for streaming operations. */ #define ZXC_MAX_THREADS 512 /** @brief Safety padding appended to buffers to tolerate overruns. */ #define ZXC_PAD_SIZE 32 /** * @brief Tail padding required on the decompression destination buffer. * * The decoder's fast path uses speculative wild-copy writes and gates * fast-loop entry on @c d_end - ZXC_DECOMPRESS_TAIL_PAD. Sizing * @c dst_capacity to @c uncompressed_size + ZXC_DECOMPRESS_TAIL_PAD * guarantees the fast path is reachable and that tail bounds checks * never spuriously reject the last literals of a valid block. * * @see zxc_decompress_block_bound() */ #define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) /** @brief Assumed CPU cache line size for alignment. */ #define ZXC_CACHE_LINE_SIZE 64 /** @brief Bitmask for cache-line alignment checks. */ #define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) /** @brief Round @p x up to the next cache-line boundary. */ #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) /** @brief File header size: Magic(4)+Version(1)+Chunk(1)+Flags(1)+Reserved(7)+CRC(2). */ #define ZXC_FILE_HEADER_SIZE 16 /** @brief Bit flag in the Flags byte indicating checksum presence (bit 7). */ #define ZXC_FILE_FLAG_HAS_CHECKSUM 0x80U /** @brief Mask for the checksum algorithm id (bits 0-3). */ #define ZXC_FILE_CHECKSUM_ALGO_MASK 0x0FU /** @brief Block header size: Type(1)+Flags(1)+Reserved(1)+CRC(1)+CompSize(4). */ #define ZXC_BLOCK_HEADER_SIZE 8 /** @brief Size of the per-block checksum field in bytes. */ #define ZXC_BLOCK_CHECKSUM_SIZE 4 /** @brief Binary size of a NUM block sub-header. */ #define ZXC_NUM_HEADER_BINARY_SIZE 16 /** @brief Binary size of a GLO block sub-header. */ #define ZXC_GLO_HEADER_BINARY_SIZE 16 /** @brief Binary size of a GHI block sub-header. */ #define ZXC_GHI_HEADER_BINARY_SIZE 16 /** @brief Binary size of a NUM chunk sub-frame header (nvals + bits + base + psize). */ #define ZXC_NUM_CHUNK_HEADER_SIZE 16 /** @brief Number of numeric values to decode in a single SIMD batch (NUM block). */ #define ZXC_NUM_DEC_BATCH 32 /** @brief Maximum number of frames that can be processed in a single compression operation (NUM * block). */ #define ZXC_NUM_FRAME_SIZE 128 /** @brief Binary size of a section descriptor (comp_size + raw_size). */ #define ZXC_SECTION_DESC_BINARY_SIZE 8 /** @brief 32-bit mask for extracting sizes from a section descriptor. */ #define ZXC_SECTION_SIZE_MASK 0xFFFFFFFFU /** @brief Number of sections in a GLO block. */ #define ZXC_GLO_SECTIONS 4 /** @brief Number of sections in a GHI block. */ #define ZXC_GHI_SECTIONS 3 /** @brief Checksum algorithm id for RapidHash (default, sole implementation). */ #define ZXC_CHECKSUM_RAPIDHASH 0 /** @brief Size of the global checksum appended after EOF block (4 bytes). */ #define ZXC_GLOBAL_CHECKSUM_SIZE 4 /** @brief File footer size: original_size(8) + global_checksum(4). */ #define ZXC_FILE_FOOTER_SIZE 12 /** @name Seekable Format Constants * @brief Seek table block appended between EOF block and footer. * * The seek table is optional (opt-in at compression time) and allows * random-access decompression by recording per-block compressed and * decompressed sizes. It uses a standard ZXC block header with * @c block_type = @ref ZXC_BLOCK_SEK. * * Detection from the end of the file: the reader derives @c num_blocks * from the file footer (total decompressed size) and file header (block size). * It then seeks backward to validate the SEK block header. * @{ */ /** @brief Per-block entry size: comp_size(4) only. decomp_size is derived * from the file header's block_size (all blocks except the last are full). */ #define ZXC_SEEK_ENTRY_SIZE 4 /** @} */ /* end of Seekable Format Constants */ /** @name GLO Token Constants * @brief 4-bit literal length / 4-bit match length / 16-bit offset. * @{ */ /** @brief Bits for Literal Length in a GLO token. */ #define ZXC_TOKEN_LIT_BITS 4 /** @brief Bits for Match Length in a GLO token. */ #define ZXC_TOKEN_ML_BITS 4 /** @brief Mask to extract Literal Length from a GLO token. */ #define ZXC_TOKEN_LL_MASK ((1U << ZXC_TOKEN_LIT_BITS) - 1) /** @brief Mask to extract Match Length from a GLO token. */ #define ZXC_TOKEN_ML_MASK ((1U << ZXC_TOKEN_ML_BITS) - 1) /** @} */ /** @name GHI Sequence Constants * @brief 8-bit literal length / 8-bit match length / 16-bit offset. * @{ */ /** @brief Bits for Literal Length in a GHI sequence. */ #define ZXC_SEQ_LL_BITS 8 /** @brief Bits for Match Length in a GHI sequence. */ #define ZXC_SEQ_ML_BITS 8 /** @brief Bits for Offset in a GHI sequence. */ #define ZXC_SEQ_OFF_BITS 16 /** @brief Mask to extract Literal Length from a GHI sequence. */ #define ZXC_SEQ_LL_MASK ((1U << ZXC_SEQ_LL_BITS) - 1) /** @brief Mask to extract Match Length from a GHI sequence. */ #define ZXC_SEQ_ML_MASK ((1U << ZXC_SEQ_ML_BITS) - 1) /** @brief Mask to extract Offset from a GHI sequence. */ #define ZXC_SEQ_OFF_MASK ((1U << ZXC_SEQ_OFF_BITS) - 1) /** @} */ /** @name Literal Stream Encoding * @{ */ /** @brief Flag bit indicating an RLE run in the literal stream (0x80). */ #define ZXC_LIT_RLE_FLAG 0x80U /** @brief Mask to extract the run/literal length (lower 7 bits). */ #define ZXC_LIT_LEN_MASK (ZXC_LIT_RLE_FLAG - 1) /** @} */ /** @name LZ77 Constants * @brief Hash table geometry, sliding window, and match parameters. * * The hash table uses a split layout with 15-bit addressing (32 768 buckets): * - `hash_positions[]`: uint32_t, stores `(epoch << offset_bits) | position` (128 KB). * - `hash_tags[]`: uint8_t, stores an 8-bit tag for fast rejection (32 KB). * Total: 160 KB. The tag table fits in L1 cache, enabling a * "filter-first" access pattern that avoids cold loads into hash_positions * on the ~60-75% of lookups where the tag mismatches. * The 64 KB sliding window allows `chain_table` to use `uint16_t`. * @{ */ /** @brief Address bits for the LZ77 hash table (2^15 = 32 768 buckets). */ #define ZXC_LZ_HASH_BITS 15 /** @brief Marsaglia multiplicative hash constant for 4-byte hashing. */ #define ZXC_LZ_HASH_PRIME1 0x2D35182DU /** @brief Marsaglia/Vigna xorshift* multiplier for 5-byte hashing. */ #define ZXC_LZ_HASH_PRIME2 0x2545F4914F6CDD1DULL /** @brief Maximum number of entries in the hash table. */ #define ZXC_LZ_HASH_SIZE (1U << ZXC_LZ_HASH_BITS) /** @brief Sliding window size (64 KB). */ #define ZXC_LZ_WINDOW_SIZE (1U << 16) /** @brief Mask for ring-buffer indexing into chain_table (power-of-two window). */ #define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U) /** @brief Minimum match length for an LZ77 match. */ #define ZXC_LZ_MIN_MATCH_LEN 5 /** @brief Base bias added to encoded offsets (stored = actual - bias). */ #define ZXC_LZ_OFFSET_BIAS 1 /** @brief Maximum allowed offset distance. */ #define ZXC_LZ_MAX_DIST (ZXC_LZ_WINDOW_SIZE - 1) /** @} */ /** @name Optimal Parser Tuning (level >= 6) * @brief Static prices and complexity guards used by the level-6 optimal * LZ77 parser DP. * @{ */ /** @brief Static price (bits) of a match token before varint extras: 1 byte * token + 2 byte offset. */ #define ZXC_OPT_MATCH_COST_BASE ((uint32_t)(3U * CHAR_BIT)) /** @brief Threshold above which `find_best_match` is skipped at intra-match * positions, keeping the parser O(N) on highly repetitive data. */ #define ZXC_OPT_LONG_MATCH_SKIP ((size_t)256) /** @brief Minimum literal count for the sample-based Huffman cost estimator * used by the optimal parser. Below this, the strided sample is too * small for the resulting code-lengths to be statistically reliable, * so the estimator falls back to RAW cost (8 bits/byte). */ #define ZXC_OPT_LIT_SAMPLE_MIN 1024 /** @} */ /** @name Hash Prime Constants * @brief Mixing primes used by internal hash functions. * @{ */ /** @brief Hash prime 1. */ #define ZXC_HASH_PRIME1 0x9E3779B97F4A7C15ULL /** @brief Hash prime 2. */ #define ZXC_HASH_PRIME2 0xD2D84A61D2D84A61ULL /** @} */ /** @name Huffman Codec Constants * @brief Length-limited canonical Huffman codec for the GLO literal stream * (active at compression level >= 6). * * On-disk section payload layout: * - @c ZXC_HUF_LENGTHS_HEADER_SIZE bytes: @c ZXC_HUF_NUM_SYMBOLS code lengths * packed two per byte (4 bits each). * - @c ZXC_HUF_STREAM_SIZES_HEADER_SIZE bytes: the first * `ZXC_HUF_NUM_STREAMS - 1` sub-stream sizes as little-endian @c uint16_t; * the last sub-stream size is derived from the enclosing section length. * - Payload: @c ZXC_HUF_NUM_STREAMS concatenated LSB-first bit-streams, * each covering an equal share of the literal indices (the last absorbs * the remainder). * * The decoder uses a single lookup table of @c ZXC_HUF_TABLE_SIZE entries * (width @c ZXC_HUF_LOOKUP_BITS) that yields 1 or 2 symbols per lookup, * feeding a `ZXC_HUF_NUM_STREAMS`-way interleaved hot loop. * @{ */ /** @brief Maximum code length, in bits. Capped well below the package-merge * algorithmic ceiling (14) to keep the decoder LUT small. */ #define ZXC_HUF_MAX_CODE_LEN 8 /** @brief Decoder LUT width: each lookup consumes this many bits and yields * 1 or 2 symbols. */ #define ZXC_HUF_LOOKUP_BITS 11 /** @brief Number of entries in the multi-symbol decoder lookup table. */ #define ZXC_HUF_TABLE_SIZE (1U << ZXC_HUF_LOOKUP_BITS) /** @brief Alphabet size: one entry per possible byte value. */ #define ZXC_HUF_NUM_SYMBOLS 256 /** @brief Interleaved bit-stream count for parallel decoding. */ #define ZXC_HUF_NUM_STREAMS 4 /** @brief Packed 4-bit code-lengths header size: two lengths per byte. */ #define ZXC_HUF_LENGTHS_HEADER_SIZE (ZXC_HUF_NUM_SYMBOLS / 2) /** @brief Sub-stream sizes header: `(ZXC_HUF_NUM_STREAMS - 1)` little-endian * @c uint16_t values; the last sub-stream size is derived from the * enclosing section length. */ #define ZXC_HUF_STREAM_SIZES_HEADER_SIZE ((int)((ZXC_HUF_NUM_STREAMS - 1) * sizeof(uint16_t))) /** @brief Total Huffman header size: lengths + sub-stream sizes. */ #define ZXC_HUF_HEADER_SIZE (ZXC_HUF_LENGTHS_HEADER_SIZE + ZXC_HUF_STREAM_SIZES_HEADER_SIZE) /** @brief Absolute floor below which Huffman cannot beat RAW even with * zero-entropy literals after the 3 % savings margin. Above this * floor, the precise size accounting at the call site decides per * block, so the threshold is corpus-agnostic. * * Derivation: the call site requires `huf_total < baseline * 31/32` * (3 % margin = `baseline >> 5`). At zero-entropy literals the * payload vanishes and `huf_total = HEADER`, giving * `N > HEADER x 32/31`. The `+30` is the standard ceiling-division * offset (`b - 1` with `b = 31`). Constants: * - 32 = inverse of the 3 % margin (`1/32`) * - 31 = `32 - 1`, the fraction kept after the margin * - 30 = `31 - 1`, ceiling-division rounding offset */ #define ZXC_HUF_MIN_LITERALS ((ZXC_HUF_HEADER_SIZE * 32 + 30) / 31) /** @brief Width of the decoder bit accumulator, in bits * (`sizeof(uint64_t) * CHAR_BIT`). */ #define ZXC_HUF_ACCUM_BITS 64 /** @brief Decoder batch size: lookups per stream between two refills. */ #define ZXC_HUF_BATCH 5 /** @brief Worst-case bits consumed per stream per batch. Must stay <= 57 so * that an 8-byte refill always brings the bit accumulator back to * >= 56 bits before the next batch. */ #define ZXC_HUF_BATCH_BITS (ZXC_HUF_BATCH * ZXC_HUF_LOOKUP_BITS) /** @brief Mask for indexing into the multi-symbol decoder lookup table. */ #define ZXC_HUF_TBL_MASK ((uint64_t)(ZXC_HUF_TABLE_SIZE - 1)) /** @brief Per-stream output headroom required to enter the batched fast loop: * each iteration speculatively writes 2 bytes per stream and runs * @c ZXC_HUF_BATCH iterations before re-checking the bound. */ #define ZXC_HUF_SAFE_MARGIN ((size_t)(2 * ZXC_HUF_BATCH)) /* Boundary package-merge work item. Each level holds at most * `2 * ZXC_HUF_NUM_SYMBOLS` of these; exposed so callers can size * pre-allocated scratch via ::ZXC_HUF_BUILD_SCRATCH_SIZE. */ typedef struct { uint32_t weight; int16_t left, right; int16_t sym; } zxc_huf_pm_item_t; /* Trace-back stack frame for the package-merge code-length recovery. */ typedef struct { int8_t lvl; int16_t idx; } zxc_huf_pm_frame_t; /** @brief Per-level item bound: at most leaves + paired packages from the * previous level. */ #define ZXC_HUF_PM_LEVEL_BOUND (2 * ZXC_HUF_NUM_SYMBOLS) /** @brief Worst-case scratch size (bytes) for ::zxc_huf_build_code_lengths. * Carved by the function into items / counts / stack regions; sized * for the worst-case alphabet (n = `ZXC_HUF_NUM_SYMBOLS`). Includes * a small alignment slack between regions. */ #define ZXC_HUF_BUILD_SCRATCH_SIZE \ ((size_t)ZXC_HUF_MAX_CODE_LEN * (size_t)ZXC_HUF_PM_LEVEL_BOUND * sizeof(zxc_huf_pm_item_t) + \ 8U + (size_t)ZXC_HUF_MAX_CODE_LEN * sizeof(int) + 8U + \ (size_t)ZXC_HUF_MAX_CODE_LEN * (size_t)ZXC_HUF_PM_LEVEL_BOUND * sizeof(zxc_huf_pm_frame_t)) /** @} */ /** @name Block Size Helpers * @brief Runtime helpers for variable block sizes. * @{ */ /** * @brief Integer log-base-2 for a 32-bit value. * @param v Must be a power of two (returns 0 for zero). * @return Floor of log2(v). */ static ZXC_ALWAYS_INLINE uint32_t zxc_log2_u32(const uint32_t v) { #ifdef _MSC_VER unsigned long index; return (v == 0) ? 0 : (_BitScanReverse(&index, v) ? index : 0); #else return (v == 0) ? 0 : (uint32_t)(31 - __builtin_clz(v)); #endif } /** * @brief Branchless bit_ceil: smallest power of two >= v, clamped to ZXC_BLOCK_SIZE_MIN. * @param[in] v Input size (must be > 0). */ static ZXC_ALWAYS_INLINE size_t zxc_block_size_ceil(const size_t v) { uint64_t x = (uint64_t)v - 1; x |= x >> 1; x |= x >> 2; x |= x >> 4; x |= x >> 8; x |= x >> 16; x |= x >> 32; x++; const size_t bs = (size_t)x; return (bs < ZXC_BLOCK_SIZE_MIN) ? ZXC_BLOCK_SIZE_MIN : bs; } /** * @brief Validates a block size. * Must be a power of two in [ZXC_BLOCK_SIZE_MIN, ZXC_BLOCK_SIZE_MAX]. * @param[in] bs Block size to validate. * @return 1 if valid, 0 otherwise. */ static ZXC_ALWAYS_INLINE int zxc_validate_block_size(const size_t bs) { return bs >= ZXC_BLOCK_SIZE_MIN && bs <= ZXC_BLOCK_SIZE_MAX && (bs & (bs - 1)) == 0; } /** @} */ /** @} */ /* end of File Format Constants */ /** * @struct zxc_lz77_params_t * @brief Search parameters for LZ77 compression levels. * * Each compression level maps to a specific set of parameters that control the * trade-off between compression speed and ratio. Higher search depths and lazy * matching improve ratio at the expense of throughput; larger step values * accelerate literal scanning but may miss short matches. */ typedef struct { /** Maximum number of candidates explored in the hash chain per position. * Higher values find better matches but increase CPU cost linearly. */ int search_depth; /** "Good enough" match length: once a match reaches this threshold the * chain walk stops immediately, avoiding wasted effort on an already * excellent match. */ int sufficient_len; /** Enable lazy matching. When set, after finding a match at position * @c ip the compressor probes @c ip+1 (and @c ip+2 for level >= 4) to * see if a longer match exists. If so, a literal is emitted and the * better match is taken instead. Improves ratio but costs extra work. */ int use_lazy; /** Maximum number of candidates explored during lazy evaluation (same * semantics as @ref search_depth but applied to the ip+1 / ip+2 probes). * Only meaningful when @ref use_lazy is non-zero. */ int lazy_attempts; /** Skip lazy evaluation when the current match length already reaches * this threshold: a match this long is unlikely to be beaten at the * next byte. Set to 0 when @ref use_lazy is disabled. */ int lazy_len_threshold; /** Base step size when advancing through unmatched literals. * 1 = test every byte (best ratio), 4 = skip aggressively (fastest). */ uint32_t step_base; /** Acceleration factor for step size: @c step = step_base + (distance >> step_shift). * A larger value keeps the step conservative (grows slowly with distance); * a smaller value ramps up quickly, skipping more in long literal runs. */ uint32_t step_shift; } zxc_lz77_params_t; /** * @brief Retrieves LZ77 compression parameters based on the specified compression level. * * This inline function returns the appropriate LZ77 parameters configuration * for the given compression level. * * @param[in] level The compression level to use for determining LZ77 parameters. * @return zxc_lz77_params_t The LZ77 parameters structure corresponding to the specified level. */ static ZXC_ALWAYS_INLINE zxc_lz77_params_t zxc_get_lz77_params(const int level) { if (level >= ZXC_LEVEL_DENSITY) return (zxc_lz77_params_t){64, 256, 0, 0, 0, 1, 8}; // search_depth, sufficient_len, use_lazy, lazy_attempts, lazy_len_threshold, step_base, // step_shift static const zxc_lz77_params_t table[6] = { {3, 16, 0, 0, 0, 4, 4}, // fallback {3, 16, 0, 0, 0, 4, 4}, // level 1 {3, 18, 0, 0, 0, 3, 6}, // level 2 {3, 16, 1, 4, 128, 1, 4}, // level 3 {3, 18, 1, 4, 128, 1, 5}, // level 4 {64, 256, 1, 16, 128, 1, 8} // level 5 }; return table[level < ZXC_LEVEL_FASTEST ? ZXC_LEVEL_FASTEST : level]; } /** * @enum zxc_block_type_t * @brief Defines the different types of data blocks supported by the ZXC * format. * * This enumeration categorizes blocks based on the compression strategy * applied: * - `ZXC_BLOCK_RAW` (0): No compression. Used when data is incompressible (high * entropy) or when compression would expand the data size. * - `ZXC_BLOCK_GLO` (1): General-purpose compression (LZ77 + Bitpacking). This * is the default for most data (text, binaries, JSON, etc.). Includes 4 sections descriptors. * - `ZXC_BLOCK_NUM` (2): Specialized compression for arrays of 32-bit integers. * Uses Delta Encoding + ZigZag + Bitpacking. * - `ZXC_BLOCK_GHI` (3): General-purpose high-velocity mode using LZ77 with advanced * techniques (lazy matching, step skipping) for maximum ratio. Includes 3 sections descriptors. * - `ZXC_BLOCK_SEK` (254): Seek table block. Contains per-block compressed/decompressed sizes * for random-access decompression. Placed between EOF block and file footer. * - `ZXC_BLOCK_EOF` (255): End of file marker. */ typedef enum { ZXC_BLOCK_RAW = 0, ZXC_BLOCK_GLO = 1, ZXC_BLOCK_NUM = 2, ZXC_BLOCK_GHI = 3, ZXC_BLOCK_SEK = 254, ZXC_BLOCK_EOF = 255 } zxc_block_type_t; /** * @enum zxc_section_encoding_t * @brief Specifies the encoding methods used for internal data sections. * * These modes determine how specific components (like literals, match lengths, * or offsets) are stored within a block. * - `ZXC_SECTION_ENCODING_RAW`: Data is stored uncompressed. * - `ZXC_SECTION_ENCODING_RLE`: Run-Length Encoding. * - `ZXC_SECTION_ENCODING_HUFFMAN`: Canonical Huffman, 4-way interleaved * sub-streams, max 11-bit codes, LSB-first. Only valid for the literal * stream (`enc_lit`) of GLO blocks. Produced exclusively at level >= 6. */ typedef enum { ZXC_SECTION_ENCODING_RAW = 0, ZXC_SECTION_ENCODING_RLE = 1, ZXC_SECTION_ENCODING_HUFFMAN = 2 } zxc_section_encoding_t; /** * @struct zxc_gnr_header_t * @brief Header specific to General (LZ-based) compression blocks. * * This header follows the main block header when the block type is GLO/GHI. It * describes the layout of sequences and literals. * * @var zxc_gnr_header_t::n_sequences * The total count of LZ sequences in the block. * @var zxc_gnr_header_t::n_literals * The total count of literal bytes. * @var zxc_gnr_header_t::enc_lit * Encoding method used for the literal stream. * @var zxc_gnr_header_t::enc_litlen * Encoding method used for the literal lengths stream. * @var zxc_gnr_header_t::enc_mlen * Encoding method used for the match lengths stream. * @var zxc_gnr_header_t::enc_off * Encoding method used for the offset stream. */ typedef struct { uint32_t n_sequences; // Number of sequences uint32_t n_literals; // Number of literals uint8_t enc_lit; // Literal encoding uint8_t enc_litlen; // Literal lengths encoding uint8_t enc_mlen; // Match lengths encoding uint8_t enc_off; // Offset encoding (Unused in Token format, kept for alignment) } zxc_gnr_header_t; /** * @struct zxc_section_desc_t * @brief Describes the size attributes of a specific data section. * * Used to track the compressed and uncompressed sizes of sub-components * (e.g., a literal stream or offset stream) within a block. */ typedef struct { uint64_t sizes; /**< Packed sizes: compressed size (low 32 bits) | raw size (high 32 bits). */ } zxc_section_desc_t; /** * @struct zxc_num_header_t * @brief Header specific to Numeric compression blocks. * * This header follows the main block header when the block type is NUM. * * @var zxc_num_header_t::n_values * The total number of numeric values encoded in the block. * @var zxc_num_header_t::frame_size * The size of the frame used for processing. */ typedef struct { uint64_t n_values; uint16_t frame_size; } zxc_num_header_t; /** * @struct zxc_bit_reader_t * @brief Internal bit reader structure for ZXC compression/decompression. * * This structure maintains the state of the bit stream reading operation. * It buffers bits from the input byte stream into an accumulator to allow * reading variable-length bit sequences. */ typedef struct { const uint8_t* ptr; /**< Pointer to the current position in the input byte stream. */ const uint8_t* end; /**< Pointer to the end of the input byte stream. */ uint64_t accum; /**< Bit accumulator holding buffered bits (64-bit buffer). */ int bits; /**< Number of valid bits currently in the accumulator. */ } zxc_bit_reader_t; /** * ============================================================================ * MEMORY & ENDIANNESS HELPERS * ============================================================================ * Functions to handle unaligned memory access and Little Endian conversion. */ /** * @brief Reads a 16-bit unsigned integer from memory in little-endian format. * * This function interprets the bytes at the given memory address as a * little-endian 16-bit integer, regardless of the host system's endianness. * It is marked as always inline for performance critical paths. * * @param[in] p Pointer to the memory location to read from. * @return The 16-bit unsigned integer value read from memory. */ static ZXC_ALWAYS_INLINE uint16_t zxc_le16(const void* p) { uint16_t v; ZXC_MEMCPY(&v, p, sizeof(v)); #ifdef ZXC_BIG_ENDIAN return ZXC_BSWAP16(v); #else return v; #endif } /** * @brief Reads a 32-bit unsigned integer from memory in little-endian format. * * This function interprets the bytes at the given pointer address as a * little-endian 32-bit integer, regardless of the host system's endianness. * It is marked as always inline for performance critical paths. * * @param[in] p Pointer to the memory location to read from. * @return The 32-bit unsigned integer value read from memory. */ static ZXC_ALWAYS_INLINE uint32_t zxc_le32(const void* p) { uint32_t v; ZXC_MEMCPY(&v, p, sizeof(v)); #ifdef ZXC_BIG_ENDIAN return ZXC_BSWAP32(v); #else return v; #endif } /** * @brief Reads a 64-bit unsigned integer from memory in little-endian format. * * This function interprets the bytes at the given memory address as a * little-endian 64-bit integer, regardless of the host system's endianness. * It is marked as always inline for performance critical paths. * * @param[in] p Pointer to the memory location to read from. * @return The 64-bit unsigned integer value read from memory. */ static ZXC_ALWAYS_INLINE uint64_t zxc_le64(const void* p) { uint64_t v; ZXC_MEMCPY(&v, p, sizeof(v)); #ifdef ZXC_BIG_ENDIAN return ZXC_BSWAP64(v); #else return v; #endif } /** * @brief Stores a 16-bit integer in memory using little-endian byte order. * * This function copies the value of a 16-bit unsigned integer to the specified * memory location. It uses memcpy to avoid strict aliasing violations and * potential unaligned access issues. * * @note This function assumes the system is little-endian or that the compiler * optimizes the memcpy to a store instruction that handles endianness if necessary * (though the implementation shown is a direct copy). * * @param[out] p Pointer to the destination memory where the value will be stored. * Must point to a valid memory region of at least 2 bytes. * @param[in] v The 16-bit unsigned integer value to store. */ static ZXC_ALWAYS_INLINE void zxc_store_le16(void* p, const uint16_t v) { #ifdef ZXC_BIG_ENDIAN const uint16_t s = ZXC_BSWAP16(v); ZXC_MEMCPY(p, &s, sizeof(s)); #else ZXC_MEMCPY(p, &v, sizeof(v)); #endif } /** * @brief Stores a 32-bit unsigned integer in little-endian format at the specified memory location. * * This function writes the 32-bit value `v` to the memory pointed to by `p`. * It uses `ZXC_MEMCPY` to ensure safe memory access, avoiding potential alignment issues * that could occur with direct pointer casting on some architectures. * * @note This function is marked as `ZXC_ALWAYS_INLINE` to minimize function call overhead. * * @param[out] p Pointer to the destination memory where the value will be stored. * @param[in] v The 32-bit unsigned integer value to store. */ static ZXC_ALWAYS_INLINE void zxc_store_le32(void* p, const uint32_t v) { #ifdef ZXC_BIG_ENDIAN const uint32_t s = ZXC_BSWAP32(v); ZXC_MEMCPY(p, &s, sizeof(s)); #else ZXC_MEMCPY(p, &v, sizeof(v)); #endif } /** * @brief Stores a 64-bit unsigned integer in little-endian format at the specified memory location. * * This function copies the 64-bit value `v` to the memory pointed to by `p`. * It uses `ZXC_MEMCPY` to ensure safe memory access, avoiding potential alignment issues * that might occur with direct pointer dereferencing on some architectures. * * @note This function assumes the system is little-endian or that the compiler optimizes * the memcpy to a store instruction that handles endianness correctly if `ZXC_MEMCPY` * is defined appropriately. * * @param[out] p Pointer to the destination memory where the value will be stored. * @param[in] v The 64-bit unsigned integer value to store. */ static ZXC_ALWAYS_INLINE void zxc_store_le64(void* p, const uint64_t v) { #ifdef ZXC_BIG_ENDIAN const uint64_t s = ZXC_BSWAP64(v); ZXC_MEMCPY(p, &s, sizeof(s)); #else ZXC_MEMCPY(p, &v, sizeof(v)); #endif } /** * @brief Computes the 1-byte checksum for block headers. * * Implementation based on Marsaglia's Xorshift (PRNG) principles. * * @param[in] p Pointer to the input data to be hashed (8 bytes) * @return uint8_t The computed hash value. */ static ZXC_ALWAYS_INLINE uint8_t zxc_hash8(const uint8_t* p) { const uint64_t v = zxc_le64(p); uint64_t h = v ^ ZXC_HASH_PRIME1; h ^= h << 13; h ^= h >> 7; h ^= h << 17; return (uint8_t)((h >> 32) ^ h); } /** * @brief Computes the 2-byte checksum for file headers. * * This function generates a hash value by reading data from the given pointer. * The result is a 16-bit hash. * Implementation based on Marsaglia's Xorshift (PRNG) principles. * * @param[in] p Pointer to the input data to be hashed (16 bytes) * @return uint16_t The computed hash value. */ static ZXC_ALWAYS_INLINE uint16_t zxc_hash16(const uint8_t* p) { const uint64_t v1 = zxc_le64(p); const uint64_t v2 = zxc_le64(p + 8); uint64_t h = v1 ^ v2 ^ ZXC_HASH_PRIME2; h ^= h << 13; h ^= h >> 7; h ^= h << 17; const uint32_t res = (uint32_t)((h >> 32) ^ h); return (uint16_t)((res >> 16) ^ res); } /** * @brief Copies 16 bytes from the source memory location to the destination memory location. * * This function is forced to be inlined and uses SIMD intrinsics when available. * SSE2 on x86/x64, NEON on ARM, or memcpy as fallback. * * @param[out] dst Pointer to the destination memory block. * @param[in] src Pointer to the source memory block. */ static ZXC_ALWAYS_INLINE void zxc_copy16(void* dst, const void* src) { #if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) // AVX2/AVX512: Single 128-bit unaligned load/store _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src)); #elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); #else ZXC_MEMCPY(dst, src, 16); #endif } /** * @brief Copies 32 bytes from source to destination using SIMD when available. * * Uses AVX2 on x86, NEON on ARM64/ARM32, or two 16-byte copies as fallback. * * @param[out] dst Pointer to the destination memory block. * @param[in] src Pointer to the source memory block. */ static ZXC_ALWAYS_INLINE void zxc_copy32(void* dst, const void* src) { #if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) // AVX2/AVX512: Single 256-bit (32 byte) unaligned load/store _mm256_storeu_si256((__m256i*)dst, _mm256_loadu_si256((const __m256i*)src)); #elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) // NEON: Two 128-bit (16 byte) unaligned load/stores vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); vst1q_u8((uint8_t*)dst + 16, vld1q_u8((const uint8_t*)src + 16)); #else ZXC_MEMCPY(dst, src, 32); #endif } /** * @brief Counts trailing zeros in a 32-bit unsigned integer. * * This function returns the number of contiguous zero bits starting from the * least significant bit (LSB). If the input is 0, it returns 32. * * It utilizes compiler-specific built-ins for GCC/Clang (`__builtin_ctz`) and * MSVC (`_BitScanForward`) for optimal performance. If no supported compiler * is detected, it falls back to a portable De Bruijn sequence implementation. * * @param[in] x The 32-bit unsigned integer to scan. * @return The number of trailing zeros (0-32). */ static ZXC_ALWAYS_INLINE int zxc_ctz32(const uint32_t x) { if (x == 0) return 32; #if defined(__GNUC__) || defined(__clang__) return __builtin_ctz(x); #elif defined(_MSC_VER) unsigned long r; _BitScanForward(&r, x); return (int)r; #else // Fallback De Bruijn (32 bits) static const int DeBruijn32[32] = {0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9}; return DeBruijn32[((uint32_t)((x & (0U - x)) * 0x077CB531U)) >> 27]; #endif } /** * @brief Counts the number of trailing zeros in a 64-bit unsigned integer. * * This function determines the number of zero bits following the least significant * one bit in the binary representation of `x`. * * @param[in] x The 64-bit unsigned integer to scan. * @return The number of trailing zeros. Returns 64 if `x` is 0. * * @note This implementation uses compiler built-ins for GCC/Clang (`__builtin_ctzll`) * and MSVC (`_BitScanForward64`) when available for optimal performance. * It falls back to a De Bruijn sequence multiplication method for other compilers. */ static ZXC_ALWAYS_INLINE int zxc_ctz64(const uint64_t x) { if (x == 0) return 64; #if defined(__GNUC__) || defined(__clang__) return __builtin_ctzll(x); #elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) unsigned long r; _BitScanForward64(&r, x); return (int)r; #elif defined(_MSC_VER) // Use two 32-bit scans to avoid fragile 64-bit De Bruijn multiplication. unsigned long r; const uint32_t lo = (uint32_t)x; if (_BitScanForward(&r, lo)) return (int)r; _BitScanForward(&r, (uint32_t)(x >> 32)); return 32 + (int)r; #else // Fallback De Bruijn for non-GCC/non-MSVC compilers static const int Debruijn64[64] = { 0, 1, 48, 2, 57, 49, 28, 3, 61, 58, 50, 42, 38, 29, 17, 4, 62, 55, 59, 36, 53, 51, 43, 22, 45, 39, 33, 30, 24, 18, 12, 5, 63, 47, 56, 27, 60, 41, 37, 16, 54, 35, 52, 21, 44, 32, 23, 11, 46, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6}; return Debruijn64[((x & (0ULL - x)) * 0x03F79D71B4CA8B09ULL) >> 58]; #endif } /** * @brief Calculates the index of the highest set bit (most significant bit) in a 32-bit integer. * * This function determines the position of the most significant bit that is set to 1. * * @param[in] n The 32-bit unsigned integer to analyze. * @return The 0-based index of the highest set bit. If n is 0, the behavior is undefined. */ static ZXC_ALWAYS_INLINE uint8_t zxc_highbit32(const uint32_t n) { #ifdef _MSC_VER unsigned long index; return (n == 0) ? 0 : (_BitScanReverse(&index, n) ? (uint8_t)(index + 1) : 0); #else return (n == 0) ? 0 : (32 - __builtin_clz(n)); #endif } /** * @brief Encodes a signed 32-bit integer using ZigZag encoding. * * ZigZag encoding maps signed integers to unsigned integers so that numbers with a small * absolute value (for instance, -1) have a small variant encoded value too. It does this * by "zig-zagging" back and forth through the positive and negative integers: * * 0 => 0 * -1 => 1 * 1 => 2 * -2 => 3 * 2 => 4 * * This is particularly useful for variable-length encoding (varint) of signed integers, * as standard varint encoding is inefficient for negative numbers (which are interpreted * as very large unsigned integers). * * @param[in] n The signed 32-bit integer to encode. * @return The ZigZag encoded unsigned 32-bit integer. */ static ZXC_ALWAYS_INLINE uint32_t zxc_zigzag_encode(const int32_t n) { return ((uint32_t)n << 1) ^ (uint32_t)(-(int32_t)((uint32_t)n >> 31)); } /** * @brief Decodes a 32-bit unsigned integer using ZigZag decoding. * * ZigZag encoding maps signed integers to unsigned integers so that numbers with a small * absolute value (for instance, -1) have a small variant encoded value too. It does this * by "zig-zagging" back and forth through the positive and negative integers: * 0 => 0, -1 => 1, 1 => 2, -2 => 3, 2 => 4, etc. * * This function reverses that process, converting the unsigned representation back into * the original signed 32-bit integer. * * @param[in] n The unsigned 32-bit integer to decode. * @return The decoded signed 32-bit integer. */ static ZXC_ALWAYS_INLINE int32_t zxc_zigzag_decode(const uint32_t n) { return (int32_t)(n >> 1) ^ -(int32_t)(n & 1); } /** * @brief Allocates aligned memory in a cross-platform manner. * * This function provides a unified interface for allocating memory with a specific * alignment requirement. It wraps `_aligned_malloc` for Windows * environments and `posix_memalign` for POSIX-compliant systems. * * @param[in] size The size of the memory block to allocate, in bytes. * @param[in] alignment The alignment value, which must be a power of two and a multiple * of `sizeof(void *)`. * @return A pointer to the allocated memory block, or NULL if the allocation fails. * The returned pointer must be freed using the corresponding aligned free function. */ void* zxc_aligned_malloc(const size_t size, const size_t alignment); /** * @brief Frees memory previously allocated with an aligned allocation function. * * This function provides a cross-platform wrapper for freeing aligned memory. * On Windows, it calls `_aligned_free`. * On other platforms, it falls back to the standard `free` function. * * @param[in] ptr A pointer to the memory block to be freed. If ptr is NULL, no operation is * performed. */ void zxc_aligned_free(void* ptr); /* * ============================================================================ * COMPRESSION CONTEXT & STRUCTS * ============================================================================ */ /* * INTERNAL API * ------------ */ /** * @brief Calculates a 32-bit hash for a given input buffer. * @param[in] input Pointer to the data buffer. * @param[in] len Length of the data in bytes. * @param[in] hash_method Checksum algorithm identifier (e.g., ZXC_CHECKSUM_RAPIDHASH). * @return The calculated 32-bit hash value. */ static ZXC_ALWAYS_INLINE uint32_t zxc_checksum(const void* RESTRICT input, const size_t len, const uint8_t hash_method) { (void)hash_method; /* single algorithm for now; extend when adding more */ const uint64_t hash = rapidhash(input, len); return (uint32_t)(hash ^ (hash >> (sizeof(uint32_t) * CHAR_BIT))); } /** * @brief Combines a running hash with a new block hash using rotate-left and XOR. * * This function updates a global checksum by rotating the current hash left by 1 bit * (with wraparound) and XORing with the new block hash. This provides a simple but * effective rolling hash that depends on the order of blocks. * * Formula: result = ((hash << 1) | (hash >> 31)) ^ block_hash * * @param[in] hash The current running hash value. * @param[in] block_hash The hash of the new block to combine. * @return The updated combined hash value. */ static ZXC_ALWAYS_INLINE uint32_t zxc_hash_combine_rotate(const uint32_t hash, const uint32_t block_hash) { return ((hash << 1) | (hash >> 31)) ^ block_hash; } /** * @brief Loads up to 7 bytes from memory in little-endian order into a uint64_t. * * This is used for partial reads at stream boundaries where fewer than 8 bytes * remain. Unlike ZXC_MEMCPY into a uint64_t (which is endian-dependent), this * function always produces a value with byte 0 in the least-significant bits. * * @param[in] p Pointer to the source bytes. * @param[in] n Number of bytes to read (must be < 8). * @return The loaded value in native host order, with bytes arranged as if * read from a little-endian stream. */ static ZXC_ALWAYS_INLINE uint64_t zxc_le_partial(const uint8_t* p, size_t n) { #ifdef ZXC_BIG_ENDIAN uint64_t v = 0; for (size_t i = 0; i < n; i++) v |= (uint64_t)p[i] << (i * CHAR_BIT); return v; #else uint64_t v = 0; n = n > sizeof(v) ? sizeof(v) : n; ZXC_MEMCPY(&v, p, n); return v; #endif } /** * @brief Initializes a bit reader structure. * * Sets up the internal state of the bit reader to read from the specified * source buffer. * * @param[out] br Pointer to the bit reader structure to initialize. * @param[in] src Pointer to the source buffer containing the data to read. * @param[in] size The size of the source buffer in bytes. */ static ZXC_ALWAYS_INLINE void zxc_br_init(zxc_bit_reader_t* RESTRICT br, const uint8_t* RESTRICT src, const size_t size) { br->ptr = src; br->end = src + size; // Safety check: ensure we have at least 8 bytes to fill the accumulator if (UNLIKELY(size < sizeof(uint64_t))) { br->accum = zxc_le_partial(src, size); br->ptr += size; br->bits = (int)(size * CHAR_BIT); } else { br->accum = zxc_le64(br->ptr); br->ptr += sizeof(uint64_t); br->bits = sizeof(uint64_t) * CHAR_BIT; } } /** * @brief Ensures that the bit reader buffer contains at least the specified * number of bits. * * This function checks if the internal buffer of the bit reader has enough bits * available to satisfy a subsequent read operation of `needed` bits. If not, it * refills the buffer from the source. * * @param[in,out] br Pointer to the bit reader context. * @param[in] needed The number of bits required to be available in the buffer. */ static ZXC_ALWAYS_INLINE void zxc_br_ensure(zxc_bit_reader_t* RESTRICT br, const int needed) { if (UNLIKELY(br->bits < needed)) { const int safe_bits = (br->bits < 0) ? 0 : br->bits; br->bits = safe_bits; // Mask out garbage bits (retain only valid existing bits) #if !defined(ZXC_DISABLE_SIMD) && defined(__BMI2__) && (defined(__x86_64__) || defined(_M_X64)) br->accum = _bzhi_u64(br->accum, safe_bits); #else br->accum &= (safe_bits < 64) ? ((1ULL << safe_bits) - 1) : ~0ULL; #endif // Calculate how many bytes we can read // We want to fill up to the accumulation capability (64 bits for uint64_t) // Bytes needed = (capacity_bits - safe_bits) / 8 const int bytes_needed = ((int)(sizeof(uint64_t) * CHAR_BIT) - safe_bits) / CHAR_BIT; // Bounds check: zxc_le64 always reads 8 bytes, so we need at least 8 const size_t bytes_left = (size_t)(br->end - br->ptr); if (UNLIKELY(bytes_left < sizeof(uint64_t))) { // Partial read (slow path / end of stream) const size_t to_read = (bytes_left < (size_t)bytes_needed) ? bytes_left : (size_t)bytes_needed; const uint64_t raw = zxc_le_partial(br->ptr, to_read); br->accum |= (safe_bits < 64) ? (raw << safe_bits) : 0; br->ptr += to_read; br->bits = safe_bits + (int)to_read * CHAR_BIT; } else { // Fast path: full 8-byte read is safe const uint64_t raw = zxc_le64(br->ptr); br->accum |= (safe_bits < 64) ? (raw << safe_bits) : 0; br->ptr += bytes_needed; br->bits = safe_bits + bytes_needed * CHAR_BIT; } } } /** * @brief Bit-packs a stream of 32-bit integers into a destination buffer. * * Compresses an array of 32-bit integers by packing them using a specified * number of bits per integer. * * @param[in] src Pointer to the source array of 32-bit integers. * @param[in] count The number of integers to pack. * @param[out] dst Pointer to the destination buffer where packed data will be * written. * @param[in] dst_cap The capacity of the destination buffer in bytes. * @param[in] bits The number of bits to use for each integer during packing. * @return int The number of bytes written to the destination buffer, or a negative * error code on failure. */ int zxc_bitpack_stream_32(const uint32_t* RESTRICT src, const size_t count, uint8_t* RESTRICT dst, const size_t dst_cap, const uint8_t bits); /** * @brief Writes a numeric header structure to a destination buffer. * * Serializes the `zxc_num_header_t` structure into the output stream. * * @param[out] dst Pointer to the destination buffer. * @param[in] rem The remaining space in the destination buffer. * @param[in] nh Pointer to the numeric header structure to write. * @return int The number of bytes written, or a negative error code if the buffer * is too small. */ int zxc_write_num_header(uint8_t* RESTRICT dst, const size_t rem, const zxc_num_header_t* RESTRICT nh); /** * @brief Reads a numeric header structure from a source buffer. * * Deserializes data from the input stream into a `zxc_num_header_t` structure. * * @param[in] src Pointer to the source buffer. * @param[in] src_size The size of the source buffer available for reading. * @param[out] nh Pointer to the numeric header structure to populate. * @return int The number of bytes read from the source, or a negative error code on * failure. */ int zxc_read_num_header(const uint8_t* RESTRICT src, const size_t src_size, zxc_num_header_t* RESTRICT nh); /** * @brief Writes a generic header and section descriptors to a destination * buffer. * * Serializes the `zxc_gnr_header_t` and an array of 4 section descriptors. * * @param[out] dst Pointer to the destination buffer. * @param[in] rem The remaining space in the destination buffer. * @param[in] gh Pointer to the generic header structure to write. * @param[in] desc Array of 4 section descriptors to write. * @return int The number of bytes written, or a negative error code if the buffer * is too small. */ int zxc_write_glo_header_and_desc(uint8_t* RESTRICT dst, const size_t rem, const zxc_gnr_header_t* RESTRICT gh, const zxc_section_desc_t desc[ZXC_GLO_SECTIONS]); /** * @brief Reads a generic header and section descriptors from a source buffer. * * Deserializes data into a `zxc_gnr_header_t` and an array of 4 section * descriptors. * * @param[in] src Pointer to the source buffer. * @param[in] len The length of the source buffer available for reading. * @param[out] gh Pointer to the generic header structure to populate. * @param[out] desc Array of 4 section descriptors to populate. * * @return int Returns ZXC_OK on success, or a negative zxc_error_t code on failure. */ int zxc_read_glo_header_and_desc(const uint8_t* RESTRICT src, const size_t len, zxc_gnr_header_t* RESTRICT gh, zxc_section_desc_t desc[ZXC_GLO_SECTIONS]); /** * @brief Writes a record header and description to the destination buffer. * * @param dst Pointer to the destination buffer where the header and description will be written. * @param rem Remaining size available in the destination buffer. * @param gh Pointer to the GNR header structure containing header information. * @param desc Array of 3 section descriptors to be written along with the header. * * @return int Returns the number of bytes written on success, or a negative error code on failure. */ int zxc_write_ghi_header_and_desc(uint8_t* RESTRICT dst, const size_t rem, const zxc_gnr_header_t* RESTRICT gh, const zxc_section_desc_t desc[ZXC_GHI_SECTIONS]); /** * @brief Reads a record header and section descriptors from a buffer. * * This function parses the source buffer to extract a general header and * up to three section descriptors from a ZXC record. * * @param[in] src Pointer to the source buffer containing the record data. * @param[in] len Length of the source buffer in bytes. * @param[out] gh Pointer to a zxc_gnr_header_t structure to store the parsed header. * @param[out] desc Array of 3 zxc_section_desc_t structures to store the parsed section * descriptors. * * @return int Returns ZXC_OK on success, or a negative zxc_error_t code on failure. */ int zxc_read_ghi_header_and_desc(const uint8_t* RESTRICT src, const size_t len, zxc_gnr_header_t* RESTRICT gh, zxc_section_desc_t desc[ZXC_GHI_SECTIONS]); /* ============================================================================ * Huffman codec for the GLO literal stream (level >= 6). * * On-disk layout, decoder geometry and tunables: see * @ref ZXC_HUF_MAX_CODE_LEN and the surrounding "Huffman Codec Constants" * group above. * ============================================================================ */ /** * @brief Build length-limited canonical Huffman code lengths from a frequency table. * * Uses the boundary package-merge algorithm capped at `ZXC_HUF_MAX_CODE_LEN`. * Symbols with `freq[i] == 0` get `code_len[i] == 0`; others receive a value * in `[1, ZXC_HUF_MAX_CODE_LEN]`. * * @param[in] freq Frequency table of length `ZXC_HUF_NUM_SYMBOLS`. * @param[out] code_len Output code-length array of length `ZXC_HUF_NUM_SYMBOLS`. * @param[in] scratch Optional caller-owned scratch buffer of at least * ::ZXC_HUF_BUILD_SCRATCH_SIZE bytes. If `NULL`, the * function allocates its own working memory and frees * it before returning. * @return `ZXC_OK` on success, negative `zxc_error_t` code on failure. */ int zxc_huf_build_code_lengths(const uint32_t* RESTRICT freq, uint8_t* RESTRICT code_len, void* RESTRICT scratch); /** * @brief Encode the literal stream into a Huffman section payload. * * Writes the 128-byte length header, the 6-byte sub-stream size table and * the 4 concatenated LSB-first bit-streams. * * @param[in] literals Source literal bytes (must not alias `dst`). * @param[in] n_literals Number of source bytes. * @param[in] code_len Per-symbol code lengths produced by * ::zxc_huf_build_code_lengths. * @param[out] dst Destination buffer for the section payload. * @param[in] dst_cap Capacity of @p dst in bytes. * @return Total bytes written on success, negative `zxc_error_t` code on failure. */ int zxc_huf_encode_section(const uint8_t* RESTRICT literals, const size_t n_literals, const uint8_t* RESTRICT code_len, uint8_t* RESTRICT dst, const size_t dst_cap); /** * @brief Decode a Huffman literal section payload of `payload_size` bytes. * * Writes exactly `n_literals` decoded bytes into @p dst. * * @param[in] payload Section payload (header + 4 sub-streams). * @param[in] payload_size Total payload length in bytes. * @param[out] dst Destination buffer (must not alias @p payload). * @param[in] n_literals Expected number of decoded bytes. * @return `ZXC_OK` on success, negative `zxc_error_t` code on failure. */ int zxc_huf_decode_section(const uint8_t* RESTRICT payload, const size_t payload_size, uint8_t* RESTRICT dst, const size_t n_literals); /** * @brief Internal wrapper function to decompress a single chunk of data. * * This function handles the decompression of a specific chunk from the source * buffer into the destination buffer using the provided compression context. It * serves as an abstraction layer over the core decompression logic. * * @param[in,out] ctx Pointer to the ZXC compression context structure containing * internal state and configuration. * @param[in] src Pointer to the source buffer containing compressed data. * @param[in] src_sz Size of the compressed data in the source buffer (in bytes). * @param[out] dst Pointer to the destination buffer where decompressed data will * be written. * @param[in] dst_cap Capacity of the destination buffer (maximum bytes that can be * written). * * @return int Returns ZXC_OK on success, or a negative zxc_error_t code on failure. * Specific error codes depend on the underlying ZXC * implementation. */ int zxc_decompress_chunk_wrapper(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap); /** * @brief Wraps the internal chunk compression logic. * * This function acts as a wrapper to compress a single chunk of data using the * provided compression context. It handles the interaction with the underlying * compression algorithm for a specific block of memory. * * @param[in,out] ctx Pointer to the ZXC compression context containing configuration * and state. * @param[in] chunk Pointer to the source buffer containing the raw data to * compress. * @param[in] src_sz The size of the source chunk in bytes. * @param[out] dst Pointer to the destination buffer where compressed data will be * written. * @param[in] dst_cap The capacity of the destination buffer (maximum bytes to write). * * @return int The number of bytes written to the destination buffer on success, * or a negative error code on failure. */ int zxc_compress_chunk_wrapper(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT chunk, const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap); /** @} */ /* end of internal */ #ifdef __cplusplus } #endif #endif // ZXC_INTERNAL_Hzxc-0.11.0/src/lib/zxc_pstream.c000066400000000000000000001267241520102567100164500ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /** * @file zxc_pstream.c * @brief Push-based, single-threaded streaming driver implementation. * * See zxc_pstream.h for the public contract. The implementation composes * the public block API (@ref zxc_compress_block / @ref zxc_decompress_block) * with the public sans-IO header helpers (@ref zxc_write_file_header / * footer, @c zxc_read_*); the only internal dependency is on shared * constants and the global-hash combine inline, pulled from zxc_internal.h. * * Both compression and decompression are structured as resumable state * machines driven by caller-provided input/output buffers * (@ref zxc_inbuf_t / @ref zxc_outbuf_t). Each call advances as much as * possible without blocking and returns a status indicating whether the * caller should drain @p out, supply more @p in, or finalise the stream. */ #include "../../include/zxc_pstream.h" #include #include #include "../../include/zxc_buffer.h" #include "../../include/zxc_constants.h" #include "../../include/zxc_error.h" #include "../../include/zxc_sans_io.h" #include "zxc_internal.h" /* ===================================================================== */ /* Compression */ /* ===================================================================== */ /** * @enum cstream_state_t * @brief Lifecycle states of the push compression stream. * * The compression state machine alternates between *staging* a chunk of * output bytes into the @c pending buffer and *draining* those bytes into * the caller's @ref zxc_outbuf_t. Forward progress is therefore always * either consuming from @c in or producing into @c out. * * @var cstream_state_t::CS_INIT * Initial state; nothing has been emitted yet. * @var cstream_state_t::CS_DRAIN_HEADER * File header staged in @c pending; draining to @p out, then transitions * to @c CS_ACCUMULATE. * @var cstream_state_t::CS_ACCUMULATE * Copying input bytes into the internal block accumulator until it is * full (or the stream is finalised). * @var cstream_state_t::CS_DRAIN_BLOCK * A full data block was just compressed; draining it to @p out, then * back to @c CS_ACCUMULATE. * @var cstream_state_t::CS_DRAIN_LAST * Draining the final partial block produced inside @ref zxc_cstream_end; * transitions to @c CS_DRAIN_EOF. * @var cstream_state_t::CS_DRAIN_EOF * Draining the EOF block; transitions to @c CS_DRAIN_FOOTER. * @var cstream_state_t::CS_DRAIN_FOOTER * Draining the file footer; transitions to @c CS_DONE. * @var cstream_state_t::CS_DONE * Finalisation complete; further @c _compress / @c _end calls are * rejected. * @var cstream_state_t::CS_ERRORED * Sticky error state; subsequent calls return the latched error code. */ typedef enum { CS_INIT = 0, CS_DRAIN_HEADER, CS_ACCUMULATE, CS_DRAIN_BLOCK, CS_DRAIN_LAST, CS_DRAIN_EOF, CS_DRAIN_FOOTER, CS_DONE, CS_ERRORED } cstream_state_t; /** * @struct zxc_cstream_s * @brief Internal state of a push compression stream. * * Owns three buffers: a fixed-size input accumulator (@c in_block, sized * to one block), a variable-size output staging area (@c pending, holding * the file header, one compressed block, the EOF block, or the file * footer), and the underlying compression context (@c cctx). * * @var zxc_cstream_s::opts * Compression options (copied from the caller at creation time). * @var zxc_cstream_s::cctx * Underlying single-block compression context. * @var zxc_cstream_s::block_size * Target block size in bytes; cached from @c opts.block_size. * @var zxc_cstream_s::in_block * Heap buffer of capacity @c block_size used to accumulate one full * uncompressed block before invoking the block compressor. * @var zxc_cstream_s::in_used * Number of valid bytes currently held in @c in_block (in * [0, @c block_size]). * @var zxc_cstream_s::pending * Heap buffer holding the next chunk of output bytes to emit * (header, compressed block, EOF marker or footer). * @var zxc_cstream_s::pending_cap * Allocated capacity of @c pending. * @var zxc_cstream_s::pending_len * Total valid bytes currently staged in @c pending. * @var zxc_cstream_s::pending_pos * Bytes already copied from @c pending to the caller's output buffer. * Drain is complete when @c pending_pos == @c pending_len. * @var zxc_cstream_s::total_in * Running count of uncompressed bytes consumed; written into the file * footer. * @var zxc_cstream_s::global_hash * Rolling per-block-trailer hash; written into the file footer when * checksums are enabled. * @var zxc_cstream_s::state * Current state machine position (see @ref cstream_state_t). * @var zxc_cstream_s::error_code * Sticky error code; valid only when @c state is @c CS_ERRORED. */ struct zxc_cstream_s { zxc_compress_opts_t opts; zxc_cctx* cctx; size_t block_size; uint8_t* in_block; size_t in_used; uint8_t* pending; size_t pending_cap; size_t pending_len; size_t pending_pos; uint64_t total_in; uint32_t global_hash; cstream_state_t state; int error_code; }; /** * @brief Latches a sticky error on the compression stream. * * Stores @p code in @c cs->error_code, transitions @c cs->state to * @c CS_ERRORED, and returns @p code. Once errored, subsequent * @ref zxc_cstream_compress / @ref zxc_cstream_end calls return the same * code without performing further work. * * @param[in,out] cs Compression stream. * @param[in] code Negative @ref zxc_error_t value to latch. * @return @p code (always negative). */ static int cs_set_error(zxc_cstream* cs, const int code) { cs->error_code = code; cs->state = CS_ERRORED; return code; } /** * @brief Compresses one full or partial accumulated block. * * Compresses the contents of @c cs->in_block into @c cs->pending, growing * the latter to @ref zxc_compress_block_bound if needed, and updates * bookkeeping (@c total_in, @c global_hash, @c in_used reset to 0). When * file-level checksums are enabled, folds the block trailer into the * rolling @c global_hash. * * @pre @c cs->in_used > 0. * * @param[in,out] cs Compression stream. * @return @ref ZXC_OK on success, negative @ref zxc_error_t on failure. */ static int cs_compress_one_block(zxc_cstream* cs) { const uint64_t bound = zxc_compress_block_bound(cs->in_used); // LCOV_EXCL_START if (UNLIKELY(bound == 0 || bound > SIZE_MAX)) return ZXC_ERROR_OVERFLOW; if (UNLIKELY(bound > cs->pending_cap)) { uint8_t* nb = (uint8_t*)realloc(cs->pending, (size_t)bound); if (UNLIKELY(!nb)) return ZXC_ERROR_MEMORY; cs->pending = nb; cs->pending_cap = (size_t)bound; } // LCOV_EXCL_STOP const int64_t csize = zxc_compress_block(cs->cctx, cs->in_block, cs->in_used, cs->pending, cs->pending_cap, &cs->opts); if (UNLIKELY(csize < 0)) return (int)csize; // LCOV_EXCL_LINE cs->pending_len = (size_t)csize; cs->pending_pos = 0; cs->total_in += cs->in_used; cs->in_used = 0; /* If checksums are on, the block trailer is the last ZXC_BLOCK_CHECKSUM_SIZE * bytes of pending; fold it into the rolling global hash. */ if (cs->opts.checksum_enabled && cs->pending_len >= ZXC_BLOCK_CHECKSUM_SIZE) { const uint32_t bh = zxc_le32(cs->pending + cs->pending_len - ZXC_BLOCK_CHECKSUM_SIZE); cs->global_hash = zxc_hash_combine_rotate(cs->global_hash, bh); } return ZXC_OK; } /** * @brief Drains staged output bytes into the caller's output buffer. * * Copies as many bytes as possible from * @c cs->pending[pending_pos..pending_len) into * @c out->dst[pos..size), advancing both cursors. * * @param[in,out] cs Compression stream. * @param[in,out] out Caller output buffer. * @return Non-zero once the @c pending buffer has been fully drained * (@c pending_pos == @c pending_len), zero otherwise. */ static int cs_drain_pending(zxc_cstream* cs, zxc_outbuf_t* out) { const size_t avail_out = out->size - out->pos; const size_t avail_pen = cs->pending_len - cs->pending_pos; const size_t n = avail_out < avail_pen ? avail_out : avail_pen; if (n) { ZXC_MEMCPY((uint8_t*)out->dst + out->pos, cs->pending + cs->pending_pos, n); out->pos += n; cs->pending_pos += n; } return cs->pending_pos == cs->pending_len; } /** * @brief Allocates and initialises a push compression stream. * * Copies @p opts into the new context, applying defaults for any zero-valued * field (@c level -> @ref ZXC_LEVEL_DEFAULT, @c block_size -> * @ref ZXC_BLOCK_SIZE_DEFAULT). Forces single-threaded operation * (@c n_threads = 0), disables progress callbacks and seekable framing * (those modes belong to the @c FILE*-based pipeline). Pre-sizes the * @c pending buffer so that the file header / footer paths never need a * realloc. * * @param[in] opts Compression options, or @c NULL for full defaults. * @return New stream owned by the caller, or @c NULL on allocation * failure / invalid option values. */ zxc_cstream* zxc_cstream_create(const zxc_compress_opts_t* opts) { zxc_cstream* cs = (zxc_cstream*)calloc(1, sizeof(*cs)); if (UNLIKELY(!cs)) return NULL; // LCOV_EXCL_LINE if (opts) cs->opts = *opts; if (cs->opts.level == 0) cs->opts.level = ZXC_LEVEL_DEFAULT; if (cs->opts.block_size == 0) cs->opts.block_size = ZXC_BLOCK_SIZE_DEFAULT; /* n_threads is ignored on this single-threaded path. */ cs->opts.n_threads = 0; cs->opts.progress_cb = NULL; cs->opts.user_data = NULL; cs->opts.seekable = 0; cs->block_size = cs->opts.block_size; cs->cctx = zxc_create_cctx(&cs->opts); // LCOV_EXCL_START if (UNLIKELY(!cs->cctx)) { free(cs); return NULL; } cs->in_block = (uint8_t*)malloc(cs->block_size); if (UNLIKELY(!cs->in_block)) { zxc_free_cctx(cs->cctx); free(cs); return NULL; } // LCOV_EXCL_STOP /* Pre-size pending so the file header path never needs realloc. */ cs->pending_cap = ZXC_FILE_HEADER_SIZE > ZXC_FILE_FOOTER_SIZE ? ZXC_FILE_HEADER_SIZE : ZXC_FILE_FOOTER_SIZE; cs->pending = (uint8_t*)malloc(cs->pending_cap); // LCOV_EXCL_START if (UNLIKELY(!cs->pending)) { free(cs->in_block); zxc_free_cctx(cs->cctx); free(cs); return NULL; } // LCOV_EXCL_STOP cs->state = CS_INIT; return cs; } /** * @brief Stages the 16-byte file header into the @c pending buffer. * * @param[in,out] cs Compression stream. * @return @ref ZXC_OK on success, negative @ref zxc_error_t on failure. */ static int cs_stage_file_header(zxc_cstream* cs) { const int w = zxc_write_file_header(cs->pending, cs->pending_cap, cs->block_size, cs->opts.checksum_enabled); if (UNLIKELY(w < 0)) return w; // LCOV_EXCL_LINE cs->pending_len = (size_t)w; cs->pending_pos = 0; return ZXC_OK; } /** * @brief Stages the 8-byte EOF block into the @c pending buffer. * * The EOF block is a regular block header with @c block_type set to * @ref ZXC_BLOCK_EOF and @c comp_size = 0; it carries no payload. * * @param[in,out] cs Compression stream. * @return @ref ZXC_OK on success, negative @ref zxc_error_t on failure. */ static int cs_stage_eof(zxc_cstream* cs) { // LCOV_EXCL_START if (UNLIKELY(ZXC_BLOCK_HEADER_SIZE > cs->pending_cap)) { uint8_t* nb = (uint8_t*)realloc(cs->pending, ZXC_BLOCK_HEADER_SIZE); if (UNLIKELY(!nb)) return ZXC_ERROR_MEMORY; cs->pending = nb; cs->pending_cap = ZXC_BLOCK_HEADER_SIZE; } // LCOV_EXCL_STOP const zxc_block_header_t eof = { .block_type = (uint8_t)ZXC_BLOCK_EOF, .block_flags = 0, .reserved = 0, .header_crc = 0, .comp_size = 0, }; const int w = zxc_write_block_header(cs->pending, cs->pending_cap, &eof); if (UNLIKELY(w < 0)) return w; // LCOV_EXCL_LINE cs->pending_len = (size_t)w; cs->pending_pos = 0; return ZXC_OK; } /** * @brief Stages the 12-byte file footer into the @c pending buffer. * * The footer carries the total uncompressed input size and (when checksums * are enabled) the global rolling hash accumulated across all data blocks. * * @param[in,out] cs Compression stream. * @return @ref ZXC_OK on success, negative @ref zxc_error_t on failure. */ static int cs_stage_footer(zxc_cstream* cs) { // LCOV_EXCL_START if (UNLIKELY(ZXC_FILE_FOOTER_SIZE > cs->pending_cap)) { uint8_t* nb = (uint8_t*)realloc(cs->pending, ZXC_FILE_FOOTER_SIZE); if (UNLIKELY(!nb)) return ZXC_ERROR_MEMORY; cs->pending = nb; cs->pending_cap = ZXC_FILE_FOOTER_SIZE; } // LCOV_EXCL_STOP const int w = zxc_write_file_footer(cs->pending, cs->pending_cap, cs->total_in, cs->global_hash, cs->opts.checksum_enabled); if (UNLIKELY(w < 0)) return w; // LCOV_EXCL_LINE cs->pending_len = (size_t)w; cs->pending_pos = 0; return ZXC_OK; } /** * @brief Releases a compression stream and all internal buffers. * * Safe to call with @c NULL. * * @param[in,out] cs Stream returned by @ref zxc_cstream_create. */ void zxc_cstream_free(zxc_cstream* cs) { if (!cs) return; free(cs->pending); free(cs->in_block); zxc_free_cctx(cs->cctx); free(cs); } /** * @brief Returns the suggested input chunk size (configured block size). * * @param[in] cs Compression stream. * @return Block size in bytes, or 0 if @p cs is @c NULL. */ size_t zxc_cstream_in_size(const zxc_cstream* cs) { return cs ? cs->block_size : 0; } /** * @brief Returns the suggested output chunk size. * * Sized to hold one full compressed block plus framing overhead, i.e. * @ref zxc_compress_block_bound applied to the configured block size. * Falls back to @c block_size when the bound overflows @c size_t. * * @param[in] cs Compression stream. * @return Suggested output buffer capacity in bytes, or 0 if @p cs is @c NULL. */ size_t zxc_cstream_out_size(const zxc_cstream* cs) { if (!cs) return 0; const uint64_t b = zxc_compress_block_bound(cs->block_size); return (b == 0 || b > SIZE_MAX) ? cs->block_size : (size_t)b; } /** * @brief Push-side entry point: feeds input and drains compressed output. * * Drives the @ref cstream_state_t machine: emits the file header on the * first call, then accumulates input until a block is full, compresses it, * and drains the result into @p out. Each call makes as much progress as * either buffer allows; the function is fully reentrant. See the public * contract in @ref zxc_cstream_compress for full semantics. * * The terminal states (@c CS_DRAIN_LAST, @c CS_DRAIN_EOF, @c CS_DRAIN_FOOTER, * @c CS_DONE, @c CS_ERRORED) are owned by @ref zxc_cstream_end; reaching * them here yields @ref ZXC_ERROR_NULL_INPUT. * * @param[in,out] cs Compression stream. * @param[in,out] out Caller output buffer. * @param[in,out] in Caller input buffer. * @return @c 0 if @p in fully consumed and nothing pending, * @c >0 number of bytes still pending (drain @p out then call again), * @c <0 a @ref zxc_error_t code. */ int64_t zxc_cstream_compress(zxc_cstream* cs, zxc_outbuf_t* out, zxc_inbuf_t* in) { if (UNLIKELY(!cs || !out || !in || in->pos > in->size || out->pos > out->size || (in->size > in->pos && !in->src) || (out->size > out->pos && !out->dst) || cs->state == CS_DONE)) { return ZXC_ERROR_NULL_INPUT; } if (UNLIKELY(cs->state == CS_ERRORED)) return cs->error_code; for (;;) { switch (cs->state) { case CS_INIT: { const int rc = cs_stage_file_header(cs); if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE cs->state = CS_DRAIN_HEADER; break; } case CS_DRAIN_HEADER: { if (!cs_drain_pending(cs, out)) return (int64_t)(cs->pending_len - cs->pending_pos); cs->state = CS_ACCUMULATE; break; } case CS_DRAIN_BLOCK: { if (!cs_drain_pending(cs, out)) return (int64_t)(cs->pending_len - cs->pending_pos); cs->state = CS_ACCUMULATE; break; } case CS_ACCUMULATE: { const size_t avail_in = in->size - in->pos; const size_t room = cs->block_size - cs->in_used; const size_t n = avail_in < room ? avail_in : room; if (n) { ZXC_MEMCPY(cs->in_block + cs->in_used, (const uint8_t*)in->src + in->pos, n); in->pos += n; cs->in_used += n; } if (cs->in_used == cs->block_size) { const int rc = cs_compress_one_block(cs); if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE cs->state = CS_DRAIN_BLOCK; break; } /* Block not yet full either in is empty or we made no progress. */ return 0; } case CS_DRAIN_LAST: case CS_DRAIN_EOF: case CS_DRAIN_FOOTER: case CS_DONE: case CS_ERRORED: /* These states are owned by _end(). */ return ZXC_ERROR_NULL_INPUT; } } } /** * @brief Finalises the stream: residual block (if any), EOF, and footer. * * Continues the same state machine as @ref zxc_cstream_compress through the * terminal states (@c CS_DRAIN_LAST -> @c CS_DRAIN_EOF -> @c CS_DRAIN_FOOTER * -> @c CS_DONE). Reentrant: when @p out fills mid-drain, returns the * number of bytes still pending and resumes from where it left off on the * next call. See the public contract in @ref zxc_cstream_end. * * @param[in,out] cs Compression stream. * @param[in,out] out Caller output buffer. * @return @c 0 once finalisation is complete (stream is now in DONE state), * @c >0 number of bytes still pending (drain and call again), * @c <0 a @ref zxc_error_t code. */ int64_t zxc_cstream_end(zxc_cstream* cs, zxc_outbuf_t* out) { if (UNLIKELY(!cs || !out || cs->state == CS_DONE)) return ZXC_ERROR_NULL_INPUT; if (UNLIKELY(cs->state == CS_ERRORED)) return cs->error_code; for (;;) { switch (cs->state) { case CS_INIT: { /* _end before any input, still need to emit file header. */ const int rc = cs_stage_file_header(cs); if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE cs->state = CS_DRAIN_HEADER; break; } case CS_DRAIN_HEADER: { if (!cs_drain_pending(cs, out)) return (int64_t)(cs->pending_len - cs->pending_pos); cs->state = CS_ACCUMULATE; break; } case CS_DRAIN_BLOCK: { /* This drain came from a full block compressed during _compress. */ if (!cs_drain_pending(cs, out)) return (int64_t)(cs->pending_len - cs->pending_pos); cs->state = CS_ACCUMULATE; break; } case CS_ACCUMULATE: { /* Compress the residual partial block (if any), then EOF + footer. */ if (cs->in_used > 0) { const int rc = cs_compress_one_block(cs); if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE cs->state = CS_DRAIN_LAST; break; } /* No residual data: go straight to EOF. */ { const int rc = cs_stage_eof(cs); if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE cs->state = CS_DRAIN_EOF; break; } } case CS_DRAIN_LAST: { if (!cs_drain_pending(cs, out)) return (int64_t)(cs->pending_len - cs->pending_pos); /* After last data block -> EOF. */ const int rc = cs_stage_eof(cs); if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE cs->state = CS_DRAIN_EOF; break; } case CS_DRAIN_EOF: { if (!cs_drain_pending(cs, out)) return (int64_t)(cs->pending_len - cs->pending_pos); const int rc = cs_stage_footer(cs); if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE cs->state = CS_DRAIN_FOOTER; break; } case CS_DRAIN_FOOTER: { if (!cs_drain_pending(cs, out)) return (int64_t)(cs->pending_len - cs->pending_pos); cs->state = CS_DONE; return 0; } case CS_DONE: case CS_ERRORED: return cs->state == CS_ERRORED ? cs->error_code : 0; } } } /* ===================================================================== */ /* Decompression */ /* ===================================================================== */ /** * @enum dstream_state_t * @brief Lifecycle states of the push decompression stream. * * The decompressor implements a frame-aware parser: file header -> N * (data block header + payload [+ optional checksum]) -> EOF block -> * optional SEK index block -> file footer. The states alternate between * *pulling* fixed- or variable-sized chunks from the caller's input, and * *emitting* the corresponding decoded output. * * @var dstream_state_t::DS_NEED_FILE_HEADER * Pulling the 16-byte file header into @c scratch. * @var dstream_state_t::DS_NEED_BLOCK_HEADER * Pulling an 8-byte block header into @c scratch. * @var dstream_state_t::DS_NEED_BLOCK_PAYLOAD * Pulling a data block payload (and optional checksum) into @c payload. * @var dstream_state_t::DS_DECODE_BLOCK * Calling the underlying block decoder on the accumulated payload. * @var dstream_state_t::DS_EMIT_DECODED * Draining decoded bytes from @c decoded into @p out. * @var dstream_state_t::DS_PEEK_TAIL * Just past the EOF block: read 8 bytes and peek to disambiguate * between an optional SEK index block and the file footer. * @var dstream_state_t::DS_DRAIN_SEK_PAYLOAD * The peeked 8 bytes were a SEK header; skipping its payload bytes. * @var dstream_state_t::DS_NEED_FOOTER_FULL * Pulling the full 12-byte file footer (used after a SEK block). * @var dstream_state_t::DS_NEED_FOOTER_REST * The 8 peeked bytes were the head of the footer; pulling the * remaining 4 bytes. * @var dstream_state_t::DS_VALIDATE_FOOTER * Validating @c total_out and the optional global hash. * @var dstream_state_t::DS_DONE * Stream fully consumed and validated. * @var dstream_state_t::DS_ERRORED * Sticky error state; subsequent calls return the latched code. */ typedef enum { DS_NEED_FILE_HEADER = 0, DS_NEED_BLOCK_HEADER, DS_NEED_BLOCK_PAYLOAD, DS_DECODE_BLOCK, DS_EMIT_DECODED, DS_PEEK_TAIL, DS_DRAIN_SEK_PAYLOAD, DS_NEED_FOOTER_FULL, DS_NEED_FOOTER_REST, DS_VALIDATE_FOOTER, DS_DONE, DS_ERRORED } dstream_state_t; /** * @struct zxc_dstream_s * @brief Internal state of a push decompression stream. * * Owns three accumulator regions: a fixed-size on-stack-style @c scratch * buffer for headers/footers/peeks, a heap @c payload buffer for variable * block payloads, and a heap @c decoded buffer that holds one decompressed * block before it is emitted to the caller. * * @var zxc_dstream_s::opts * Decompression options (copied at creation time). * @var zxc_dstream_s::inner * Underlying single-block decompression context. * @var zxc_dstream_s::inner_initialized * Non-zero once @c inner has been initialised; gates the matching * @ref zxc_cctx_free call at teardown. * @var zxc_dstream_s::block_size * Block size declared by the file header; 0 until the header is parsed. * @var zxc_dstream_s::file_has_checksum * File-level checksum flag declared by the file header. * @var zxc_dstream_s::scratch * Generic accumulator for fixed-size frames; sized for the largest * (file header = 16 bytes). * @var zxc_dstream_s::scratch_used * Number of bytes currently held in @c scratch. * @var zxc_dstream_s::scratch_need * Target number of bytes for the current accumulation phase. * @var zxc_dstream_s::payload * Heap buffer holding one block: header + compressed payload + optional * checksum. * @var zxc_dstream_s::payload_cap * Allocated capacity of @c payload. * @var zxc_dstream_s::payload_used * Number of valid bytes currently in @c payload. * @var zxc_dstream_s::payload_need * Target byte count for the current payload phase * (= header size + comp_size + checksum size). * @var zxc_dstream_s::decoded * Heap buffer holding the decoded output of one block (sized for the * wild-copy fast path: @c block_size + @ref ZXC_PAD_SIZE). * @var zxc_dstream_s::decoded_cap * Allocated capacity of @c decoded. * @var zxc_dstream_s::decoded_size * Real number of decoded bytes in @c decoded after the last decode. * @var zxc_dstream_s::decoded_pos * Bytes already emitted from @c decoded; drain complete when * @c decoded_pos == @c decoded_size. * @var zxc_dstream_s::cur_bh * Parsed block header for the block currently being processed. * @var zxc_dstream_s::sek_remaining * Bytes left to skip from a SEK block payload (only used in * @c DS_DRAIN_SEK_PAYLOAD). * @var zxc_dstream_s::total_out * Cumulative decompressed output size; cross-checked against the * file footer. * @var zxc_dstream_s::global_hash * Rolling per-block-trailer hash; cross-checked against the file * footer when checksums are enabled. * @var zxc_dstream_s::state * Current state machine position (see @ref dstream_state_t). * @var zxc_dstream_s::error_code * Sticky error code; valid only when @c state is @c DS_ERRORED. */ struct zxc_dstream_s { zxc_decompress_opts_t opts; zxc_cctx_t inner; int inner_initialized; size_t block_size; int file_has_checksum; uint8_t scratch[32]; size_t scratch_used; size_t scratch_need; uint8_t* payload; size_t payload_cap; size_t payload_used; size_t payload_need; uint8_t* decoded; size_t decoded_cap; size_t decoded_size; size_t decoded_pos; zxc_block_header_t cur_bh; size_t sek_remaining; uint64_t total_out; uint32_t global_hash; dstream_state_t state; int error_code; }; /** * @brief Latches a sticky error on the decompression stream. * * Stores @p code in @c ds->error_code, transitions @c ds->state to * @c DS_ERRORED, and returns @p code. Once errored, subsequent * @ref zxc_dstream_decompress calls return the same code without * performing further work. * * @param[in,out] ds Decompression stream. * @param[in] code Negative @ref zxc_error_t value to latch. * @return @p code (always negative). */ static int ds_set_error(zxc_dstream* ds, const int code) { ds->error_code = code; ds->state = DS_ERRORED; return code; } /** * @brief Pulls up to @c (scratch_need - scratch_used) bytes from @p in. * * Used to accumulate fixed-size frames (file header, block header, footer, * EOF tail peek) into the inline @c scratch buffer. * * @param[in,out] ds Decompression stream. * @param[in,out] in Caller input buffer; @c pos is advanced. * @return @c 1 once @c scratch holds exactly @c scratch_need bytes, * @c 0 otherwise (need more input). */ static int ds_pull_scratch(zxc_dstream* ds, zxc_inbuf_t* in) { const size_t want = ds->scratch_need - ds->scratch_used; const size_t avail = in->size - in->pos; const size_t n = want < avail ? want : avail; if (n) { ZXC_MEMCPY(ds->scratch + ds->scratch_used, (const uint8_t*)in->src + in->pos, n); in->pos += n; ds->scratch_used += n; } return ds->scratch_used == ds->scratch_need; } /** * @brief Same as @ref ds_pull_scratch but pulls into the heap @c payload buffer. * * Used to accumulate the variable-size compressed block payload * (header + comp_size [+ checksum]). * * @param[in,out] ds Decompression stream. * @param[in,out] in Caller input buffer; @c pos is advanced. * @return @c 1 once @c payload holds exactly @c payload_need bytes, * @c 0 otherwise (need more input). */ static int ds_pull_payload(zxc_dstream* ds, zxc_inbuf_t* in) { const size_t want = ds->payload_need - ds->payload_used; const size_t avail = in->size - in->pos; const size_t n = want < avail ? want : avail; if (n) { ZXC_MEMCPY(ds->payload + ds->payload_used, (const uint8_t*)in->src + in->pos, n); in->pos += n; ds->payload_used += n; } return ds->payload_used == ds->payload_need; } /** * @brief Allocates and initialises a push decompression stream. * * Copies @p opts into the new context. The internal multi-threading, * progress-callback, and seekable knobs are forced off (those modes belong * to the @c FILE*-based pipeline). @c block_size, @c file_has_checksum, * and the @c payload / @c decoded buffers are filled in lazily once the * file header is parsed. * * @param[in] opts Decompression options, or @c NULL for full defaults. * @return New stream owned by the caller, or @c NULL on allocation failure. */ zxc_dstream* zxc_dstream_create(const zxc_decompress_opts_t* opts) { zxc_dstream* ds = (zxc_dstream*)calloc(1, sizeof(*ds)); if (UNLIKELY(!ds)) return NULL; // LCOV_EXCL_LINE if (opts) ds->opts = *opts; ds->opts.n_threads = 0; ds->opts.progress_cb = NULL; ds->opts.user_data = NULL; ds->state = DS_NEED_FILE_HEADER; ds->scratch_need = ZXC_FILE_HEADER_SIZE; return ds; } /** * @brief Releases a decompression stream and all internal buffers. * * Safe to call with @c NULL. * * @param[in,out] ds Stream returned by @ref zxc_dstream_create. */ void zxc_dstream_free(zxc_dstream* ds) { if (!ds) return; free(ds->payload); free(ds->decoded); if (ds->inner_initialized) zxc_cctx_free(&ds->inner); free(ds); } /** * @brief Returns 1 iff the stream has reached @c DS_DONE. * * @param[in] ds Decompression stream. * @return @c 1 if DONE, @c 0 otherwise (including errored). */ int zxc_dstream_finished(const zxc_dstream* ds) { return (ds && ds->state == DS_DONE) ? 1 : 0; } /** * @brief Returns the suggested input chunk size for the decompressor. * * Before the file header is parsed the call returns * @ref ZXC_BLOCK_SIZE_DEFAULT; afterwards it returns the maximal compressed * block size derived from the negotiated @c block_size. * * @param[in] ds Decompression stream. * @return Suggested input buffer capacity in bytes, or 0 if @p ds is @c NULL. */ size_t zxc_dstream_in_size(const zxc_dstream* ds) { if (!ds) return 0; if (ds->block_size == 0) return ZXC_BLOCK_SIZE_DEFAULT; const uint64_t b = zxc_compress_block_bound(ds->block_size); return (b == 0 || b > SIZE_MAX) ? ds->block_size : (size_t)b; } /** * @brief Returns the suggested output chunk size for the decompressor. * * Equals the negotiated @c block_size; before the file header is parsed, * returns @ref ZXC_BLOCK_SIZE_DEFAULT. * * @param[in] ds Decompression stream. * @return Suggested output buffer capacity in bytes, or 0 if @p ds is @c NULL. */ size_t zxc_dstream_out_size(const zxc_dstream* ds) { if (!ds) return 0; return ds->block_size == 0 ? ZXC_BLOCK_SIZE_DEFAULT : ds->block_size; } /** * @brief Drains @c ds->decoded[decoded_pos..decoded_size) into @p out. * * Updates @c ds->total_out by the number of bytes copied and, when * @p produced is non-NULL, accumulates the same count into @c *produced * (used by the outer state machine to compute the per-call return value). * * @param[in,out] ds Decompression stream. * @param[in,out] out Caller output buffer. * @param[in,out] produced Optional running count of bytes produced this call. * @return @c 1 once @c decoded is fully drained, @c 0 otherwise. */ static int ds_drain_decoded(zxc_dstream* ds, zxc_outbuf_t* out, size_t* produced) { const size_t avail_out = out->size - out->pos; const size_t avail_dec = ds->decoded_size - ds->decoded_pos; const size_t n = avail_out < avail_dec ? avail_out : avail_dec; if (n) { ZXC_MEMCPY((uint8_t*)out->dst + out->pos, ds->decoded + ds->decoded_pos, n); out->pos += n; ds->decoded_pos += n; ds->total_out += n; if (produced) *produced += n; } return ds->decoded_pos == ds->decoded_size; } /** * @brief Handles the @c DS_NEED_FILE_HEADER state. * * Pulls the 16-byte file header into @c scratch, parses it via * @ref zxc_read_file_header, and lazily allocates the @c payload and * @c decoded buffers (sized from the negotiated @c block_size). The * @c decoded buffer is over-allocated by @ref ZXC_PAD_SIZE bytes to absorb * wild-copy overflow from the inner decoder. Initialises the underlying * decompression context and transitions to @c DS_NEED_BLOCK_HEADER. * * @param[in,out] ds Decompression stream. * @param[in,out] in Caller input buffer. * @return @c 1 if more input is needed, @c 0 to continue the outer loop, * negative @ref zxc_error_t on validation/allocation failure. */ static int ds_handle_need_file_header(zxc_dstream* ds, zxc_inbuf_t* in) { if (!ds_pull_scratch(ds, in)) return 1; size_t bs = 0; int has_csum = 0; const int rc = zxc_read_file_header(ds->scratch, ds->scratch_used, &bs, &has_csum); if (UNLIKELY(rc != ZXC_OK)) return ds_set_error(ds, rc); // LCOV_EXCL_LINE ds->block_size = bs; ds->file_has_checksum = has_csum; /* Allocate payload + decoded buffers now that block_size is known. */ const uint64_t pb = zxc_compress_block_bound(ds->block_size); // LCOV_EXCL_START if (UNLIKELY(pb == 0 || pb > SIZE_MAX)) return ds_set_error(ds, ZXC_ERROR_OVERFLOW); // LCOV_EXCL_STOP ds->payload_cap = (size_t)pb; ds->payload = (uint8_t*)malloc(ds->payload_cap); /* Decoded buffer is sized for the wild-copy fast path: block_size + * ZXC_PAD_SIZE; same pattern as the buffer API uses internally. Real * decoded payload lives in [0..decoded_size); the trailing PAD bytes * hold wild-copy overflow we never emit. */ ds->decoded_cap = ds->block_size + ZXC_PAD_SIZE; ds->decoded = (uint8_t*)malloc(ds->decoded_cap); // LCOV_EXCL_START if (UNLIKELY(!ds->payload || !ds->decoded)) return ds_set_error(ds, ZXC_ERROR_MEMORY); if (UNLIKELY(zxc_cctx_init(&ds->inner, ds->block_size, 0, 0, ds->file_has_checksum && ds->opts.checksum_enabled) != ZXC_OK)) { return ds_set_error(ds, ZXC_ERROR_MEMORY); } // LCOV_EXCL_STOP ds->inner_initialized = 1; ds->state = DS_NEED_BLOCK_HEADER; ds->scratch_used = 0; ds->scratch_need = ZXC_BLOCK_HEADER_SIZE; return 0; } /** * @brief Handles the @c DS_NEED_BLOCK_HEADER state. * * Pulls 8 bytes into @c scratch and parses them as a block header. If the * block is an EOF block, transitions to @c DS_PEEK_TAIL to disambiguate * between an optional SEK index and the file footer. Otherwise, validates * the announced @c comp_size against the @c payload buffer capacity, copies * the parsed header into @c payload (the underlying decoder expects header * + body + optional checksum as a single contiguous frame), and transitions * to @c DS_NEED_BLOCK_PAYLOAD. * * @param[in,out] ds Decompression stream. * @param[in,out] in Caller input buffer. * @return @c 1 if more input is needed, @c 0 to continue the outer loop, * negative @ref zxc_error_t on validation/allocation failure. */ static int ds_handle_need_block_header(zxc_dstream* ds, zxc_inbuf_t* in) { if (!ds_pull_scratch(ds, in)) return 1; const int rc = zxc_read_block_header(ds->scratch, ds->scratch_used, &ds->cur_bh); if (UNLIKELY(rc != ZXC_OK)) return ds_set_error(ds, rc); // LCOV_EXCL_LINE if (ds->cur_bh.block_type == (uint8_t)ZXC_BLOCK_EOF) { /* EOF block: comp_size must be 0; no payload, no checksum. */ if (UNLIKELY(ds->cur_bh.comp_size != 0)) return ds_set_error(ds, ZXC_ERROR_BAD_BLOCK_SIZE); ds->state = DS_PEEK_TAIL; ds->scratch_used = 0; ds->scratch_need = ZXC_BLOCK_HEADER_SIZE; /* sniff */ return 0; } /* Normal data block: read comp_size [+ ZXC_BLOCK_CHECKSUM_SIZE if file-level checksums]. */ const uint64_t need = (uint64_t)ds->cur_bh.comp_size + (ds->file_has_checksum ? (uint64_t)ZXC_BLOCK_CHECKSUM_SIZE : 0u); if (UNLIKELY(need > ds->payload_cap)) return ds_set_error(ds, ZXC_ERROR_BAD_BLOCK_SIZE); /* Feed the full block (header + payload + opt csum) to zxc_decompress_block, * so prefix with the 8-byte header we just parsed. */ ZXC_MEMCPY(ds->payload, ds->scratch, ZXC_BLOCK_HEADER_SIZE); ds->payload_used = ZXC_BLOCK_HEADER_SIZE; ds->payload_need = (size_t)need + ZXC_BLOCK_HEADER_SIZE; // LCOV_EXCL_START if (UNLIKELY(ds->payload_need > ds->payload_cap)) { /* grow */ uint8_t* nb = (uint8_t*)realloc(ds->payload, ds->payload_need); if (UNLIKELY(!nb)) return ds_set_error(ds, ZXC_ERROR_MEMORY); ds->payload = nb; ds->payload_cap = ds->payload_need; } // LCOV_EXCL_STOP ds->state = DS_NEED_BLOCK_PAYLOAD; return 0; } /** * @brief Push-side entry point: feeds compressed input and drains decoded output. * * Drives the @ref dstream_state_t machine: file header, then per-block * (header + payload + optional checksum) -> decode -> emit, repeated until * the EOF block, optional SEK index block, and file footer have been parsed * and validated. Each call makes as much progress as @p in and @p out * allow; the function is fully reentrant. See the public contract in * @ref zxc_dstream_decompress for full semantics. * * @par End of stream * Once @c DS_VALIDATE_FOOTER succeeds, the stream is in @c DS_DONE; further * calls return @c 0 without consuming input. * * @par Errors * On any negative return the stream becomes errored (sticky); subsequent * calls keep returning the same code until @ref zxc_dstream_free. * * @param[in,out] ds Decompression stream. * @param[in,out] out Caller output buffer. * @param[in,out] in Caller input buffer. * @return @c >0 number of decompressed bytes written into @p out this call, * @c 0 stream complete (DONE) or no progress possible, * @c <0 a @ref zxc_error_t code. */ int64_t zxc_dstream_decompress(zxc_dstream* ds, zxc_outbuf_t* out, zxc_inbuf_t* in) { if (UNLIKELY(!ds || !out || !in || in->pos > in->size || out->pos > out->size || (in->size > in->pos && !in->src) || (out->size > out->pos && !out->dst))) { return ZXC_ERROR_NULL_INPUT; } if (UNLIKELY(ds->state == DS_ERRORED)) return ds->error_code; if (UNLIKELY(ds->state == DS_DONE)) return 0; size_t produced = 0; for (;;) { switch (ds->state) { case DS_NEED_FILE_HEADER: { const int rc = ds_handle_need_file_header(ds, in); if (rc == 1) return (int64_t)produced; if (rc < 0) return rc; break; } case DS_NEED_BLOCK_HEADER: { const int rc = ds_handle_need_block_header(ds, in); if (rc == 1) return (int64_t)produced; if (rc < 0) return rc; break; } case DS_NEED_BLOCK_PAYLOAD: { if (!ds_pull_payload(ds, in)) return (int64_t)produced; ds->state = DS_DECODE_BLOCK; break; } case DS_DECODE_BLOCK: { const int dsz = zxc_decompress_chunk_wrapper( &ds->inner, ds->payload, ds->payload_used, ds->decoded, ds->decoded_cap); if (UNLIKELY(dsz < 0)) return ds_set_error(ds, dsz); ds->decoded_size = (size_t)dsz; ds->decoded_pos = 0; /* If file-level checksum verification is enabled, fold this * block's trailer into the rolling global hash (last * ZXC_BLOCK_CHECKSUM_SIZE bytes of the *raw* block). */ if (ds->opts.checksum_enabled && ds->file_has_checksum && ds->payload_used >= ZXC_BLOCK_CHECKSUM_SIZE) { const uint32_t bh = zxc_le32(ds->payload + ds->payload_used - ZXC_BLOCK_CHECKSUM_SIZE); ds->global_hash = zxc_hash_combine_rotate(ds->global_hash, bh); } ds->state = DS_EMIT_DECODED; break; } case DS_EMIT_DECODED: { const int done = ds_drain_decoded(ds, out, &produced); if (!done) return (int64_t)produced; ds->state = DS_NEED_BLOCK_HEADER; ds->scratch_used = 0; ds->scratch_need = ZXC_BLOCK_HEADER_SIZE; break; } case DS_PEEK_TAIL: { if (!ds_pull_scratch(ds, in)) return (int64_t)produced; /* Try to interpret as a block header (SEK). */ zxc_block_header_t peek; const int sek_rc = zxc_read_block_header(ds->scratch, ds->scratch_used, &peek); if (sek_rc == ZXC_OK && peek.block_type == (uint8_t)ZXC_BLOCK_SEK) { /* SEK block: skip its payload (peek.comp_size bytes). */ ds->sek_remaining = (size_t)peek.comp_size; ds->state = DS_DRAIN_SEK_PAYLOAD; break; } /* Not SEK -> these 8 bytes are the first 8 of the 12-byte footer. */ ds->state = DS_NEED_FOOTER_REST; ds->scratch_need = ZXC_FILE_FOOTER_SIZE; /* keep first 8, want 4 more */ break; } case DS_DRAIN_SEK_PAYLOAD: { const size_t avail = in->size - in->pos; const size_t n = avail < ds->sek_remaining ? avail : ds->sek_remaining; in->pos += n; ds->sek_remaining -= n; if (ds->sek_remaining > 0) return (int64_t)produced; ds->state = DS_NEED_FOOTER_FULL; ds->scratch_used = 0; ds->scratch_need = ZXC_FILE_FOOTER_SIZE; break; } case DS_NEED_FOOTER_REST: case DS_NEED_FOOTER_FULL: { if (!ds_pull_scratch(ds, in)) return (int64_t)produced; ds->state = DS_VALIDATE_FOOTER; break; } case DS_VALIDATE_FOOTER: { const uint64_t declared = zxc_le64(ds->scratch); if (UNLIKELY(declared != ds->total_out)) return ds_set_error(ds, ZXC_ERROR_CORRUPT_DATA); if (ds->opts.checksum_enabled && ds->file_has_checksum) { const uint32_t fh = zxc_le32(ds->scratch + sizeof(uint64_t)); if (UNLIKELY(fh != ds->global_hash)) return ds_set_error(ds, ZXC_ERROR_BAD_CHECKSUM); } ds->state = DS_DONE; return (int64_t)produced; } case DS_DONE: case DS_ERRORED: return ds->state == DS_ERRORED ? ds->error_code : (int64_t)produced; } } } zxc-0.11.0/src/lib/zxc_seekable.c000066400000000000000000001005021520102567100165320ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /** * @file zxc_seekable.c * @brief Seekable archive reader (random-access decompression) and seek table writer. * * The seek table is a standard ZXC block (type = ZXC_BLOCK_SEK) appended * between the EOF block and the file footer. It records the compressed size * of every block (decompressed sizes are derived from the header's block_size), * enabling O(1) lookup + O(block_size) decompression for any byte range. * * On-disk layout of a SEK block: * * [Block Header (8B)] block_type=SEK, block_flags=0, comp_size=N*4 * [N x Entry (4B)] comp_size(u32 LE) per block * * Detection from end of file: * 1. Read file header (first 16 bytes) => block_size * 2. Read file footer (last 12 bytes) => total_decompressed_size * 3. Derive num_blocks = ceil(total_decomp / block_size) * 4. Compute seek block size, read backward to the block header * 5. Validate block_type == ZXC_BLOCK_SEK */ #include "../../include/zxc_seekable.h" #include "../../include/zxc_error.h" #include "../../include/zxc_sans_io.h" #include "zxc_internal.h" /* ========================================================================= */ /* Platform Threading & I/O Layer */ /* ========================================================================= */ // LCOV_EXCL_START - Windows platform layer, not reachable on POSIX CI #if defined(_WIN32) #include /* _get_osfhandle, _fileno */ #include /* _beginthreadex */ #include /* MSVC does not provide fseeko/ftello - map to 64-bit equivalents */ #if defined(_MSC_VER) && !defined(fseeko) #define fseeko _fseeki64 #define ftello _ftelli64 #endif /* Map POSIX threading primitives to Windows equivalents */ typedef HANDLE zxc_thread_t; typedef struct { void* (*func)(void*); void* arg; } zxc_seek_thread_arg_t; static unsigned __stdcall zxc_seek_thread_entry(void* p) { zxc_seek_thread_arg_t* a = (zxc_seek_thread_arg_t*)p; void* (*f)(void*) = a->func; void* arg = a->arg; free(a); f(arg); return 0; } static int zxc_seek_thread_create(zxc_thread_t* t, void* (*fn)(void*), void* arg) { zxc_seek_thread_arg_t* wrapper = malloc(sizeof(zxc_seek_thread_arg_t)); if (UNLIKELY(!wrapper)) return ZXC_ERROR_MEMORY; wrapper->func = fn; wrapper->arg = arg; uintptr_t handle = _beginthreadex(NULL, 0, zxc_seek_thread_entry, wrapper, 0, NULL); if (UNLIKELY(handle == 0)) { free(wrapper); return ZXC_ERROR_MEMORY; } *t = (HANDLE)handle; return 0; } static void zxc_seek_thread_join(zxc_thread_t t) { WaitForSingleObject(t, INFINITE); CloseHandle(t); } static int zxc_seek_get_num_procs(void) { SYSTEM_INFO si; GetSystemInfo(&si); return (int)si.dwNumberOfProcessors; } /** * @brief Thread-safe positional read (Windows). * * Uses ReadFile + Overlapped to read at a specific offset without moving the * file pointer, making it safe for concurrent access from multiple threads. */ static int zxc_seek_pread(HANDLE hFile, void* buf, size_t count, uint64_t offset) { OVERLAPPED ov; ZXC_MEMSET(&ov, 0, sizeof(ov)); ov.Offset = (DWORD)(offset & 0xFFFFFFFF); ov.OffsetHigh = (DWORD)(offset >> 32); DWORD bytes_read = 0; if (!ReadFile(hFile, buf, (DWORD)count, &bytes_read, &ov)) return ZXC_ERROR_IO; return (bytes_read == (DWORD)count) ? (int)count : ZXC_ERROR_IO; } // LCOV_EXCL_STOP #else /* POSIX */ #include #include typedef pthread_t zxc_thread_t; static int zxc_seek_thread_create(zxc_thread_t* t, void* (*fn)(void*), void* arg) { return pthread_create(t, NULL, fn, arg) == 0 ? 0 : ZXC_ERROR_MEMORY; } static void zxc_seek_thread_join(zxc_thread_t t) { pthread_join(t, NULL); } static int zxc_seek_get_num_procs(void) { const long n = sysconf(_SC_NPROCESSORS_ONLN); return (n > 0) ? (int)n : 1; } /** * @brief Thread-safe positional read (POSIX). * * Uses pread() which reads at a given offset without modifying the file * descriptor's current position, making it inherently thread-safe. */ static int zxc_seek_pread(int fd, void* buf, size_t count, uint64_t offset) { const ssize_t r = pread(fd, buf, count, (off_t)offset); return (r == (ssize_t)count) ? (int)count : ZXC_ERROR_IO; } #endif /* _WIN32 */ /* ========================================================================= */ /* Seek Table Writer */ /* ========================================================================= */ size_t zxc_seek_table_size(const uint32_t num_blocks) { return ZXC_BLOCK_HEADER_SIZE + (size_t)num_blocks * ZXC_SEEK_ENTRY_SIZE; } int64_t zxc_write_seek_table(uint8_t* dst, const size_t dst_capacity, const uint32_t* comp_sizes, const uint32_t num_blocks) { if (UNLIKELY(num_blocks > UINT32_MAX / ZXC_SEEK_ENTRY_SIZE)) return ZXC_ERROR_OVERFLOW; const size_t total = zxc_seek_table_size(num_blocks); if (UNLIKELY(dst_capacity < total)) return ZXC_ERROR_DST_TOO_SMALL; if (UNLIKELY(!dst || !comp_sizes)) return ZXC_ERROR_NULL_INPUT; const uint32_t payload_size = num_blocks * ZXC_SEEK_ENTRY_SIZE; /* Write standard ZXC block header */ const zxc_block_header_t bh = { .block_type = ZXC_BLOCK_SEK, .block_flags = 0, .reserved = 0, .comp_size = payload_size}; const int hdr_res = zxc_write_block_header(dst, dst_capacity, &bh); if (UNLIKELY(hdr_res < 0)) return hdr_res; uint8_t* p = dst + hdr_res; /* Write entries: comp_size(4) only */ for (uint32_t i = 0; i < num_blocks; i++) { zxc_store_le32(p, comp_sizes[i]); p += sizeof(uint32_t); } return (int64_t)(p - dst); } /* ========================================================================= */ /* Seekable Reader (Opaque Handle) */ /* ========================================================================= */ struct zxc_seekable_s { /* Source - exactly one is non-NULL */ const uint8_t* src; uint64_t src_size; FILE* file; /* Native file descriptor for thread-safe pread() I/O */ #if defined(_WIN32) HANDLE native_handle; /* from GetOSFileHandle or _get_osfhandle */ #else int fd; /* from fileno() */ #endif /* Parsed seek table */ uint32_t num_blocks; uint32_t* comp_sizes; /* array[num_blocks] */ uint64_t* comp_offsets; /* prefix-sum: byte offset in compressed file per block */ uint64_t total_decomp; /* total decompressed size (from footer) */ /* File header info - block_size is always a power of 2 in [4KB, 2MB], * fits in 21 bits. */ uint32_t block_size; int file_has_checksums; /* Reusable decompression context (single-threaded path only) */ zxc_cctx_t dctx; int dctx_initialized; }; /** * @brief Parses the seek table from raw bytes at the end of the archive. * * Detection (backward from end): * 1. Read file header => block_size * 2. Read file footer => total_decomp_size * 3. Derive num_blocks = ceil(total_decomp_size / block_size) * 4. Compute expected seek block position, validate block_type == SEK * 5. Read comp_sizes; derive decomp_sizes from block_size */ static zxc_seekable* zxc_seekable_parse(const uint8_t* data, const size_t data_size) { /* Minimum: file_header(16) + eof_block(8) + seek_block_header(8) * + file_footer(12) = 44 */ const size_t MIN_SEEKABLE_SIZE = ZXC_FILE_HEADER_SIZE + ZXC_BLOCK_HEADER_SIZE + ZXC_BLOCK_HEADER_SIZE + ZXC_FILE_FOOTER_SIZE; if (UNLIKELY(data_size < MIN_SEEKABLE_SIZE)) return NULL; /* Step 1: validate file header => block_size */ size_t block_size_sz = 0; int file_has_chk = 0; if (UNLIKELY(zxc_read_file_header(data, data_size, &block_size_sz, &file_has_chk) != ZXC_OK)) return NULL; const uint32_t block_size = (uint32_t)block_size_sz; if (UNLIKELY(block_size == 0)) return NULL; // LCOV_EXCL_LINE /* Step 2: read total decompressed size from the file footer */ const uint8_t* const footer_ptr = data + data_size - ZXC_FILE_FOOTER_SIZE; const uint64_t total_decomp = zxc_le64(footer_ptr); /* A value of 0 means empty file - no seek table */ if (UNLIKELY(total_decomp == 0)) return NULL; /* Step 3: derive num_blocks = ceil(total_decomp / block_size) */ const uint64_t num_blocks_64 = (total_decomp + block_size - 1) / block_size; if (UNLIKELY(num_blocks_64 > UINT32_MAX)) return NULL; const uint32_t num_blocks = (uint32_t)num_blocks_64; /* Step 4: compute seek block position and validate. */ const uint64_t entries_total_64 = num_blocks_64 * ZXC_SEEK_ENTRY_SIZE; if (UNLIKELY(entries_total_64 > SIZE_MAX - ZXC_BLOCK_HEADER_SIZE)) return NULL; const size_t entries_total = (size_t)entries_total_64; const size_t seek_block_total = ZXC_BLOCK_HEADER_SIZE + entries_total; if (UNLIKELY(seek_block_total + ZXC_FILE_FOOTER_SIZE > data_size)) return NULL; const uint8_t* const seek_block_start = data + data_size - ZXC_FILE_FOOTER_SIZE - seek_block_total; if (UNLIKELY(seek_block_start < data)) return NULL; /* Read and validate SEK block header */ zxc_block_header_t bh; if (UNLIKELY(zxc_read_block_header(seek_block_start, seek_block_total, &bh) != ZXC_OK)) return NULL; if (UNLIKELY(bh.block_type != ZXC_BLOCK_SEK)) return NULL; if (UNLIKELY(bh.comp_size != (uint32_t)entries_total)) return NULL; /* Step 5: allocate handle and parse entries */ zxc_seekable* const s = (zxc_seekable*)calloc(1, sizeof(zxc_seekable)); // LCOV_EXCL_START if (UNLIKELY(!s)) return NULL; // LCOV_EXCL_STOP s->num_blocks = num_blocks; s->block_size = block_size; s->file_has_checksums = file_has_chk; s->src = data; s->src_size = (uint64_t)data_size; /* Allocate arrays */ s->comp_sizes = (uint32_t*)calloc(num_blocks, sizeof(uint32_t)); s->comp_offsets = (uint64_t*)calloc((size_t)num_blocks + 1, sizeof(uint64_t)); // LCOV_EXCL_START if (UNLIKELY(!s->comp_sizes || !s->comp_offsets)) { zxc_seekable_free(s); return NULL; } // LCOV_EXCL_STOP s->total_decomp = total_decomp; /* Parse comp_sizes and build compressed prefix sums. * Validate each comp_size against data_size to prevent prefix-sum overflow * and out-of-bounds reads during decompression. */ const uint8_t* ep = seek_block_start + ZXC_BLOCK_HEADER_SIZE; uint64_t comp_acc = ZXC_FILE_HEADER_SIZE; /* blocks start after file header */ for (uint32_t i = 0; i < num_blocks; i++) { s->comp_sizes[i] = zxc_le32(ep); ep += sizeof(uint32_t); /* Reject entries below minimum (block header) or larger than the file */ if (UNLIKELY(s->comp_sizes[i] < ZXC_BLOCK_HEADER_SIZE || s->comp_sizes[i] > (uint64_t)data_size)) { zxc_seekable_free(s); return NULL; } s->comp_offsets[i] = comp_acc; comp_acc += s->comp_sizes[i]; /* Reject if cumulative offset exceeds file size (inconsistent table) */ if (UNLIKELY(comp_acc > (uint64_t)data_size)) { zxc_seekable_free(s); return NULL; } } s->comp_offsets[num_blocks] = comp_acc; /* Verify prefix-sum lands exactly at the EOF block position. * Expected layout: [header 16][data blocks][EOF 8][SEK block][footer 12] * So comp_acc (end of data blocks) + EOF(8) == seek_block_start. */ const uint64_t expected_eof_offset = (uint64_t)(seek_block_start - data) - ZXC_BLOCK_HEADER_SIZE; if (UNLIKELY(comp_acc != expected_eof_offset)) { zxc_seekable_free(s); return NULL; } /* Validate that an actual EOF block header exists at the computed offset */ if (UNLIKELY(comp_acc + ZXC_BLOCK_HEADER_SIZE > data_size)) { zxc_seekable_free(s); return NULL; } zxc_block_header_t eof_bh; if (UNLIKELY(zxc_read_block_header(data + comp_acc, ZXC_BLOCK_HEADER_SIZE, &eof_bh) != ZXC_OK || eof_bh.block_type != ZXC_BLOCK_EOF)) { zxc_seekable_free(s); return NULL; } return s; } zxc_seekable* zxc_seekable_open(const void* src, const size_t src_size) { if (UNLIKELY(!src || src_size == 0)) return NULL; return zxc_seekable_parse((const uint8_t*)src, src_size); } zxc_seekable* zxc_seekable_open_file(FILE* f) { if (UNLIKELY(!f)) return NULL; const long long saved_pos = ftello(f); if (UNLIKELY(saved_pos < 0)) return NULL; // LCOV_EXCL_LINE // LCOV_EXCL_START if (UNLIKELY(fseeko(f, 0, SEEK_END) != 0)) { fseeko(f, saved_pos, SEEK_SET); return NULL; } // LCOV_EXCL_STOP const long long file_size = ftello(f); // LCOV_EXCL_START if (UNLIKELY(file_size <= 0)) { fseeko(f, saved_pos, SEEK_SET); return NULL; } // LCOV_EXCL_STOP /* For simplicity and correctness: read the file into memory. * The seek table parsing needs the file header. */ if ((size_t)file_size <= 64 * 1024 * 1024) { /* File <= 64 MB: read it all into memory */ uint8_t* const full = (uint8_t*)malloc((size_t)file_size); // LCOV_EXCL_START if (UNLIKELY(!full)) { fseeko(f, saved_pos, SEEK_SET); return NULL; } if (UNLIKELY(fseeko(f, 0, SEEK_SET) != 0 || fread(full, 1, (size_t)file_size, f) != (size_t)file_size)) { free(full); fseeko(f, saved_pos, SEEK_SET); return NULL; } // LCOV_EXCL_STOP fseeko(f, saved_pos, SEEK_SET); zxc_seekable* const s = zxc_seekable_parse(full, (size_t)file_size); if (s) { s->src = NULL; s->src_size = (uint64_t)file_size; s->file = f; #if defined(_WIN32) s->native_handle = (HANDLE)(intptr_t)_get_osfhandle(_fileno(f)); #else s->fd = fileno(f); #endif } free(full); return s; } // LCOV_EXCL_START - large file path (>64MB), not reachable in unit tests /* Large file: read header + footer separately */ uint8_t header[ZXC_FILE_HEADER_SIZE]; if (UNLIKELY(fseeko(f, 0, SEEK_SET) != 0 || fread(header, 1, ZXC_FILE_HEADER_SIZE, f) != ZXC_FILE_HEADER_SIZE)) { fseeko(f, saved_pos, SEEK_SET); return NULL; } size_t bs_sz = 0; int fhc = 0; if (UNLIKELY(zxc_read_file_header(header, ZXC_FILE_HEADER_SIZE, &bs_sz, &fhc) != ZXC_OK)) { fseeko(f, saved_pos, SEEK_SET); return NULL; } const uint32_t bs = (uint32_t)bs_sz; /* Read footer (12 bytes) to get total_decomp_size */ uint8_t footer_buf[ZXC_FILE_FOOTER_SIZE]; if (UNLIKELY(fseeko(f, file_size - (long long)ZXC_FILE_FOOTER_SIZE, SEEK_SET) != 0 || fread(footer_buf, 1, ZXC_FILE_FOOTER_SIZE, f) != ZXC_FILE_FOOTER_SIZE)) { fseeko(f, saved_pos, SEEK_SET); return NULL; } const uint64_t total_decomp = zxc_le64(footer_buf); if (UNLIKELY(total_decomp == 0)) { fseeko(f, saved_pos, SEEK_SET); return NULL; } /* Derive num_blocks = ceil(total_decomp / block_size). * Guard against uint32_t overflow. */ const uint64_t num_blocks_64 = (total_decomp + bs - 1) / bs; if (UNLIKELY(num_blocks_64 > UINT32_MAX)) { fseeko(f, saved_pos, SEEK_SET); return NULL; } const uint32_t num_blocks = (uint32_t)num_blocks_64; /* Guard against size_t multiplication overflow. */ const uint64_t entries_total_64 = (uint64_t)num_blocks * ZXC_SEEK_ENTRY_SIZE; if (UNLIKELY(entries_total_64 > SIZE_MAX - ZXC_BLOCK_HEADER_SIZE)) { fseeko(f, saved_pos, SEEK_SET); return NULL; } /* Read the full seek block */ const size_t seek_block_total = ZXC_BLOCK_HEADER_SIZE + (size_t)entries_total_64; uint8_t* const seek_buf = (uint8_t*)malloc(seek_block_total); if (UNLIKELY(!seek_buf)) { fseeko(f, saved_pos, SEEK_SET); return NULL; } const long long seek_offset = file_size - (long long)ZXC_FILE_FOOTER_SIZE - (long long)seek_block_total; if (UNLIKELY(seek_offset < 0 || fseeko(f, seek_offset, SEEK_SET) != 0 || fread(seek_buf, 1, seek_block_total, f) != seek_block_total)) { free(seek_buf); fseeko(f, saved_pos, SEEK_SET); return NULL; } fseeko(f, saved_pos, SEEK_SET); /* Validate block header */ zxc_block_header_t bh; if (UNLIKELY(zxc_read_block_header(seek_buf, seek_block_total, &bh) != ZXC_OK) || bh.block_type != ZXC_BLOCK_SEK || bh.comp_size != (uint32_t)entries_total_64) { free(seek_buf); return NULL; } /* Build seekable handle */ zxc_seekable* const s = (zxc_seekable*)calloc(1, sizeof(zxc_seekable)); if (UNLIKELY(!s)) { free(seek_buf); return NULL; } s->file = f; s->src = NULL; #if defined(_WIN32) s->native_handle = (HANDLE)(intptr_t)_get_osfhandle(_fileno(f)); #else s->fd = fileno(f); #endif s->src_size = (uint64_t)file_size; s->num_blocks = num_blocks; s->block_size = bs; s->file_has_checksums = fhc; s->comp_sizes = (uint32_t*)calloc(num_blocks, sizeof(uint32_t)); s->comp_offsets = (uint64_t*)calloc((size_t)num_blocks + 1, sizeof(uint64_t)); if (UNLIKELY(!s->comp_sizes || !s->comp_offsets)) { free(seek_buf); zxc_seekable_free(s); return NULL; } s->total_decomp = total_decomp; const uint8_t* ep = seek_buf + ZXC_BLOCK_HEADER_SIZE; uint64_t comp_acc = ZXC_FILE_HEADER_SIZE; for (uint32_t i = 0; i < num_blocks; i++) { s->comp_sizes[i] = zxc_le32(ep); ep += sizeof(uint32_t); /* Reject entries larger than the entire file */ if (UNLIKELY(s->comp_sizes[i] > (uint64_t)file_size)) { free(seek_buf); zxc_seekable_free(s); return NULL; } s->comp_offsets[i] = comp_acc; comp_acc += s->comp_sizes[i]; } s->comp_offsets[num_blocks] = comp_acc; free(seek_buf); return s; // LCOV_EXCL_STOP } uint32_t zxc_seekable_get_num_blocks(const zxc_seekable* s) { return s ? s->num_blocks : 0; } uint64_t zxc_seekable_get_decompressed_size(const zxc_seekable* s) { return s ? s->total_decomp : 0; } uint32_t zxc_seekable_get_block_comp_size(const zxc_seekable* s, const uint32_t block_idx) { if (UNLIKELY(!s || block_idx >= s->num_blocks)) return 0; return s->comp_sizes[block_idx]; } uint32_t zxc_seekable_get_block_decomp_size(const zxc_seekable* s, const uint32_t block_idx) { if (UNLIKELY(!s || block_idx >= s->num_blocks)) return 0; const uint64_t start = (uint64_t)block_idx * (uint64_t)s->block_size; const uint64_t remaining = s->total_decomp - start; return (remaining >= (uint64_t)s->block_size) ? s->block_size : (uint32_t)remaining; } /* ========================================================================= */ /* Random-Access Decompression */ /* ========================================================================= */ /** @brief O(1) block lookup: block_index = offset / block_size. */ static uint32_t zxc_seek_find_block(const uint32_t block_size, const uint64_t offset) { return (uint32_t)(offset / (uint64_t)block_size); } /** @brief O(1) decompressed offset for block @p idx. */ static uint64_t zxc_seek_decomp_offset(const uint32_t block_size, const uint32_t idx) { return (uint64_t)idx * (uint64_t)block_size; } /** @brief O(1) decompressed size for block @p idx. */ static uint32_t zxc_seek_decomp_size(const uint32_t block_size, const uint64_t total_decomp, const uint32_t idx) { const uint64_t start = (uint64_t)idx * (uint64_t)block_size; const uint64_t remaining = total_decomp - start; return (remaining >= (uint64_t)block_size) ? block_size : (uint32_t)remaining; } /** * @brief Reads a compressed block from buffer or file. */ static int zxc_seek_read_block(const zxc_seekable* s, const uint32_t block_idx, uint8_t* buf, const size_t buf_cap) { const uint64_t off = s->comp_offsets[block_idx]; const uint32_t csz = s->comp_sizes[block_idx]; if (UNLIKELY(csz > buf_cap)) return ZXC_ERROR_DST_TOO_SMALL; if (s->src) { /* Buffer mode */ if (UNLIKELY(off + csz > s->src_size)) return ZXC_ERROR_SRC_TOO_SMALL; ZXC_MEMCPY(buf, s->src + off, csz); } else if (s->file) { /* File mode */ // LCOV_EXCL_START if (UNLIKELY(fseeko(s->file, (long long)off, SEEK_SET) != 0 || fread(buf, 1, csz, s->file) != csz)) return ZXC_ERROR_IO; // LCOV_EXCL_STOP } else { return ZXC_ERROR_NULL_INPUT; // LCOV_EXCL_LINE } return (int)csz; } int64_t zxc_seekable_decompress_range(zxc_seekable* s, void* dst, const size_t dst_capacity, const uint64_t offset, const size_t len) { if (UNLIKELY(!s || !dst)) return ZXC_ERROR_NULL_INPUT; if (UNLIKELY(len == 0)) return 0; if (UNLIKELY(dst_capacity < len)) return ZXC_ERROR_DST_TOO_SMALL; if (UNLIKELY(offset + len > s->total_decomp)) return ZXC_ERROR_SRC_TOO_SMALL; /* Initialize decompression context on first use */ if (!s->dctx_initialized) { // LCOV_EXCL_START if (UNLIKELY(zxc_cctx_init(&s->dctx, (size_t)s->block_size, 0, 0, 0) != ZXC_OK)) return ZXC_ERROR_MEMORY; // LCOV_EXCL_STOP s->dctx_initialized = 1; } /* Ensure work buffer is large enough */ const size_t work_sz = (size_t)s->block_size + ZXC_PAD_SIZE; if (s->dctx.work_buf_cap < work_sz) { free(s->dctx.work_buf); s->dctx.work_buf = (uint8_t*)malloc(work_sz); if (UNLIKELY(!s->dctx.work_buf)) return ZXC_ERROR_MEMORY; // LCOV_EXCL_LINE s->dctx.work_buf_cap = work_sz; } /* Find block range - O(1) division */ const uint32_t blk_start = zxc_seek_find_block(s->block_size, offset); const uint32_t blk_end = zxc_seek_find_block(s->block_size, offset + len - 1); uint8_t* out = (uint8_t*)dst; size_t remaining = len; /* Allocate read buffer for compressed blocks */ size_t max_comp = 0; for (uint32_t bi = blk_start; bi <= blk_end; bi++) { if (s->comp_sizes[bi] > max_comp) max_comp = s->comp_sizes[bi]; } uint8_t* const read_buf = (uint8_t*)malloc(max_comp + ZXC_PAD_SIZE); if (UNLIKELY(!read_buf)) return ZXC_ERROR_MEMORY; // LCOV_EXCL_LINE for (uint32_t bi = blk_start; bi <= blk_end; bi++) { /* Read compressed block data */ const int read_res = zxc_seek_read_block(s, bi, read_buf, max_comp + ZXC_PAD_SIZE); if (UNLIKELY(read_res < 0)) { free(read_buf); return read_res; } /* Decompress the block */ const int dec_res = zxc_decompress_chunk_wrapper(&s->dctx, read_buf, (size_t)read_res, s->dctx.work_buf, work_sz); if (UNLIKELY(dec_res < 0)) { free(read_buf); return dec_res; } /* Calculate which portion of this block's decompressed data we need */ const uint64_t blk_decomp_start = zxc_seek_decomp_offset(s->block_size, bi); const size_t skip = (offset > blk_decomp_start) ? (size_t)(offset - blk_decomp_start) : 0; if (UNLIKELY((size_t)dec_res < skip)) { free(read_buf); return ZXC_ERROR_CORRUPT_DATA; } const size_t avail = (size_t)dec_res - skip; const size_t copy = (avail < remaining) ? avail : remaining; ZXC_MEMCPY(out, s->dctx.work_buf + skip, copy); out += copy; remaining -= copy; } free(read_buf); return (int64_t)len; } /* ========================================================================= */ /* Multi-Threaded Random-Access Decompression (Fork-Join) */ /* ========================================================================= */ /** * @brief Per-block job descriptor for multi-threaded decompression. * * Each worker thread receives a pointer to one of these, performs the read + * decompress + memcpy sequence, and writes the result code into @c result. * The main thread inspects @c result after join. */ typedef struct { const zxc_seekable* s; /* shared handle (read-only) */ uint32_t block_idx; /* block to decompress */ uint8_t* dst; /* output pointer within caller's buffer */ size_t skip; /* bytes to skip at start of decompressed block */ size_t copy_len; /* bytes to copy into dst */ int result; /* 0 = OK, < 0 = error */ } zxc_seek_mt_job_t; /** * @brief Thread-safe block read using pread (for file mode) or memcpy (buffer mode). */ static int zxc_seek_read_block_mt(const zxc_seekable* s, const uint32_t block_idx, uint8_t* buf, const size_t buf_cap) { const uint64_t off = s->comp_offsets[block_idx]; const uint32_t csz = s->comp_sizes[block_idx]; if (UNLIKELY(csz > buf_cap)) return ZXC_ERROR_DST_TOO_SMALL; if (s->src) { /* Buffer mode - memcpy is inherently thread-safe on const data */ if (UNLIKELY(off + csz > s->src_size)) return ZXC_ERROR_SRC_TOO_SMALL; ZXC_MEMCPY(buf, s->src + off, csz); } else if (s->file) { /* File mode - use pread for concurrent, lock-free reads */ #if defined(_WIN32) const int r = zxc_seek_pread(s->native_handle, buf, csz, off); #else const int r = zxc_seek_pread(s->fd, buf, csz, off); #endif if (UNLIKELY(r < 0)) return r; } else { return ZXC_ERROR_NULL_INPUT; // LCOV_EXCL_LINE } return (int)csz; } /** * @brief Worker thread entry point for multi-threaded seekable decompression. * * Each worker: * 1. Allocates a thread-local decompression context. * 2. Reads the compressed block via pread (thread-safe). * 3. Decompresses into a local work buffer. * 4. Copies the requested sub-range into the caller's output buffer. */ static void* zxc_seek_mt_worker(void* arg) { zxc_seek_mt_job_t* const job = (zxc_seek_mt_job_t*)arg; const zxc_seekable* const s = job->s; const uint32_t bi = job->block_idx; /* Thread-local decompression context (mode=0 for decompress-only) */ zxc_cctx_t dctx; // LCOV_EXCL_START if (UNLIKELY(zxc_cctx_init(&dctx, (size_t)s->block_size, 0, 0, 0) != ZXC_OK)) { job->result = ZXC_ERROR_MEMORY; return NULL; } // LCOV_EXCL_STOP /* Allocate work buffer for decompressed output */ const size_t work_sz = (size_t)s->block_size + ZXC_PAD_SIZE; dctx.work_buf = (uint8_t*)malloc(work_sz); // LCOV_EXCL_START if (UNLIKELY(!dctx.work_buf)) { zxc_cctx_free(&dctx); job->result = ZXC_ERROR_MEMORY; return NULL; } // LCOV_EXCL_STOP dctx.work_buf_cap = work_sz; /* Read compressed block */ const uint32_t csz = s->comp_sizes[bi]; uint8_t* const read_buf = (uint8_t*)malloc(csz + ZXC_PAD_SIZE); // LCOV_EXCL_START if (UNLIKELY(!read_buf)) { zxc_cctx_free(&dctx); job->result = ZXC_ERROR_MEMORY; return NULL; } // LCOV_EXCL_STOP const int read_res = zxc_seek_read_block_mt(s, bi, read_buf, csz + ZXC_PAD_SIZE); // LCOV_EXCL_START if (UNLIKELY(read_res < 0)) { free(read_buf); zxc_cctx_free(&dctx); job->result = read_res; return NULL; } // LCOV_EXCL_STOP /* Decompress */ const int dec_res = zxc_decompress_chunk_wrapper(&dctx, read_buf, (size_t)read_res, dctx.work_buf, work_sz); free(read_buf); // LCOV_EXCL_START if (UNLIKELY(dec_res < 0)) { zxc_cctx_free(&dctx); job->result = dec_res; return NULL; } if (UNLIKELY((size_t)dec_res < job->skip + job->copy_len)) { zxc_cctx_free(&dctx); job->result = ZXC_ERROR_CORRUPT_DATA; return NULL; } // LCOV_EXCL_STOP /* Copy the requested portion directly into the caller's output buffer */ ZXC_MEMCPY(job->dst, dctx.work_buf + job->skip, job->copy_len); zxc_cctx_free(&dctx); job->result = 0; return NULL; } int64_t zxc_seekable_decompress_range_mt(zxc_seekable* s, void* dst, const size_t dst_capacity, const uint64_t offset, const size_t len, int n_threads) { if (UNLIKELY(!s || !dst)) return ZXC_ERROR_NULL_INPUT; if (UNLIKELY(len == 0)) return 0; if (UNLIKELY(dst_capacity < len)) return ZXC_ERROR_DST_TOO_SMALL; if (UNLIKELY(offset + len > s->total_decomp)) return ZXC_ERROR_SRC_TOO_SMALL; /* Find block range - O(1) division */ const uint32_t blk_start = zxc_seek_find_block(s->block_size, offset); const uint32_t blk_end = zxc_seek_find_block(s->block_size, offset + len - 1); const uint32_t num_jobs = blk_end - blk_start + 1; /* Auto-detect thread count (0 = use all available cores) */ if (n_threads == 0) n_threads = zxc_seek_get_num_procs(); /* Fallback to single-threaded path for trivial cases */ if (n_threads <= 1 || num_jobs <= 1) { return zxc_seekable_decompress_range(s, dst, dst_capacity, offset, len); } /* Cap threads to number of blocks and max limit */ if ((uint32_t)n_threads > num_jobs) n_threads = (int)num_jobs; if (n_threads > ZXC_MAX_THREADS) n_threads = ZXC_MAX_THREADS; /* Allocate job descriptors */ zxc_seek_mt_job_t* const jobs = (zxc_seek_mt_job_t*)calloc(num_jobs, sizeof(zxc_seek_mt_job_t)); if (UNLIKELY(!jobs)) return ZXC_ERROR_MEMORY; // LCOV_EXCL_LINE /* Plan jobs: compute skip, copy_len, and dst pointer for each block */ uint8_t* out = (uint8_t*)dst; size_t remaining = len; for (uint32_t i = 0; i < num_jobs; i++) { const uint32_t bi = blk_start + i; const uint64_t blk_decomp_start = zxc_seek_decomp_offset(s->block_size, bi); const size_t skip = (offset > blk_decomp_start) ? (size_t)(offset - blk_decomp_start) : 0; const size_t blk_decomp_sz = zxc_seek_decomp_size(s->block_size, s->total_decomp, bi); if (UNLIKELY(blk_decomp_sz < skip)) { free(jobs); return ZXC_ERROR_CORRUPT_DATA; } const size_t avail = blk_decomp_sz - skip; const size_t copy = (avail < remaining) ? avail : remaining; jobs[i].s = s; jobs[i].block_idx = bi; jobs[i].dst = out; jobs[i].skip = skip; jobs[i].copy_len = copy; jobs[i].result = 0; out += copy; remaining -= copy; } /* Launch worker threads (fork phase) */ zxc_thread_t* const threads = (zxc_thread_t*)malloc((size_t)n_threads * sizeof(zxc_thread_t)); // LCOV_EXCL_START if (UNLIKELY(!threads)) { free(jobs); return ZXC_ERROR_MEMORY; } // LCOV_EXCL_STOP /* * Distribute jobs across threads round-robin style. * If num_jobs > n_threads, some threads handle multiple blocks sequentially. * We process jobs in waves: spawn n_threads at a time, join, repeat. */ int error = 0; uint32_t job_idx = 0; while (job_idx < num_jobs && !error) { const int wave_size = ((int)(num_jobs - job_idx) < n_threads) ? (int)(num_jobs - job_idx) : n_threads; int launched = 0; for (int t = 0; t < wave_size; t++) { // LCOV_EXCL_START if (zxc_seek_thread_create(&threads[t], zxc_seek_mt_worker, &jobs[job_idx + t]) != 0) { /* Failed to create thread - mark remaining jobs as errors */ for (uint32_t j = job_idx + (uint32_t)t; j < num_jobs; j++) jobs[j].result = ZXC_ERROR_MEMORY; error = 1; break; } // LCOV_EXCL_STOP launched++; } /* Join phase */ for (int t = 0; t < launched; t++) { zxc_seek_thread_join(threads[t]); if (jobs[job_idx + t].result < 0) error = 1; } job_idx += (uint32_t)launched; } free(threads); /* Check for errors */ int64_t result = (int64_t)len; if (error) { for (uint32_t i = 0; i < num_jobs; i++) { if (jobs[i].result < 0) { result = (int64_t)jobs[i].result; break; } } } free(jobs); return result; } void zxc_seekable_free(zxc_seekable* s) { if (!s) return; if (s->dctx_initialized) zxc_cctx_free(&s->dctx); free(s->comp_sizes); free(s->comp_offsets); free(s); } zxc-0.11.0/tests/000077500000000000000000000000001520102567100135365ustar00rootroot00000000000000zxc-0.11.0/tests/fuzz_decompress.c000066400000000000000000000012221520102567100171210ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ #include #include #include #include "../include/zxc_buffer.h" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { static uint8_t out_buf[4 << 20]; /* 4 MiB */ size_t out_capacity = sizeof(out_buf); uint64_t expected_size = zxc_get_decompressed_size(data, size); if (expected_size > 0 && expected_size <= out_capacity) out_capacity = (size_t)expected_size; zxc_decompress(data, size, out_buf, out_capacity, 0); return 0; }zxc-0.11.0/tests/fuzz_pstream.c000066400000000000000000000244151520102567100164410ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /** * @file fuzz_pstream.c * @brief Fuzzer for the push-based streaming API (zxc_cstream / zxc_dstream). * * The push API is a state machine driven entirely by caller-supplied byte * slices: each call may fully consume input, partially consume it, fill the * output buffer mid-block, or hit an error. These resumption paths are * unique to pstream and are NOT exercised by fuzz_decompress.c (which feeds * the whole blob in a single call). * * Strategy: derive an input-chunk size and an output-drain size from the * fuzzer header bytes so libFuzzer explores byte content AND chunk * boundaries jointly. Phases: * * 1. Compress fuzzed data via zxc_cstream_*, draining the output in small * chunks (forces re-entry into the compressor state machine). * 2. Decompress the result via zxc_dstream_decompress, feeding the input * one chunk at a time, again with a small output drain buffer. Assert * bit-exact roundtrip. * 3. Feed the *raw* fuzzer input directly to a fresh dstream, varying chunk * sizes - this is the parser/state-machine fuzzing surface that catches * malformed-frame, truncation, and sticky-error bugs. */ #include #include #include #include #include #include "../include/zxc_pstream.h" /* Cap aligned with the maximum supported block size (2 MiB). */ #define FUZZ_PSTREAM_MAX_INPUT (2 << 20) /* 2 MiB */ /* Tiny output buffer: forces the compressor / decompressor into multi-round * draining, which is precisely the resumption code we want to fuzz. */ #define FUZZ_PSTREAM_OUT_CAP 256 static size_t derive_chunk_size(uint8_t b) { /* Map [0..255] -> [1..512] with a non-linear distribution biased toward * small values (small chunks stress the state machine the most). */ if (b == 0) return 1; if (b < 64) return (size_t)b; /* 1..63 */ if (b < 192) return (size_t)b * 2; /* 128..382 */ return (size_t)b + 256; /* 448..511 */ } /* ---------------------------------------------------------------- * * Decompress `comp` into `out`, feeding input in chunks of `in_chunk` and * draining output in chunks of FUZZ_PSTREAM_OUT_CAP. * * Returns the total number of decompressed bytes written, or a negative * zxc error code on failure. Stops early (no error) if the output buffer * is exhausted before the stream is finished. * ---------------------------------------------------------------- */ static int64_t drip_decompress(const uint8_t* comp, size_t csize, uint8_t* out, size_t out_cap, size_t in_chunk, int checksum_enabled) { zxc_decompress_opts_t opts = {.checksum_enabled = checksum_enabled}; zxc_dstream* ds = zxc_dstream_create(&opts); if (!ds) return -1; size_t in_pos = 0; size_t out_total = 0; while (in_pos < csize || !zxc_dstream_finished(ds)) { if (out_total >= out_cap) { /* Caller's buffer full - stop without asserting. */ zxc_dstream_free(ds); return (int64_t)out_total; } const size_t feed = (csize - in_pos < in_chunk) ? (csize - in_pos) : in_chunk; size_t window = out_cap - out_total; if (window > FUZZ_PSTREAM_OUT_CAP) window = FUZZ_PSTREAM_OUT_CAP; zxc_inbuf_t in = {.src = comp + in_pos, .size = feed, .pos = 0}; zxc_outbuf_t ob = {.dst = out + out_total, .size = window, .pos = 0}; const size_t before_in = in.pos; const size_t before_out = ob.pos; const int64_t r = zxc_dstream_decompress(ds, &ob, &in); if (r < 0) { zxc_dstream_free(ds); return r; } out_total += ob.pos; in_pos += in.pos; /* No progress and no more input: prevent infinite loop on truncation. */ if (in.pos == before_in && ob.pos == before_out && in_pos == csize) break; } zxc_dstream_free(ds); return (int64_t)out_total; } int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { if (size < 3) return 0; /* Save raw input for the parser-fuzzing phase. */ const uint8_t* const raw_data = data; const size_t raw_size = size; /* Header bytes drive pstream-specific axes. data[0] is reserved (the * compression level is intentionally fixed to 1, see copts below); it * is still consumed so corpus byte offsets stay stable. */ const int checksum_enabled = data[1] & 1; const size_t in_chunk = derive_chunk_size(data[2]); data += 3; size -= 3; if (size == 0 || size > FUZZ_PSTREAM_MAX_INPUT) return 0; /* Persistent buffers - reused across iterations to reduce allocator * pressure (same pattern as fuzz_seekable.c). */ static uint8_t* comp_buf = NULL; static size_t comp_cap = 0; static uint8_t* round_buf = NULL; static size_t round_cap = 0; /* ------------------------------------------------------------------ */ /* Phase 1: Push compression in chunks */ /* ------------------------------------------------------------------ */ /* Level forced to 1: pstream's state-machine surface (framing, drain, * resumption, sticky errors) is independent of compression strength, * and buffer-API fuzzers (roundtrip, decompress) already exercise the * level dimension. Cheaper compression = more iterations/sec. */ zxc_compress_opts_t copts = { .level = 1, .checksum_enabled = checksum_enabled, }; zxc_cstream* cs = zxc_cstream_create(&copts); if (!cs) return 0; /* Sized so the compressor never runs out - bound is a safe upper limit. */ /* Use the public buffer-API bound function via a local extern declaration * to avoid pulling zxc_buffer.h (kept narrow on purpose). */ extern uint64_t zxc_compress_bound(size_t input_size); const uint64_t bound64 = zxc_compress_bound(size); if (bound64 == 0 || bound64 > SIZE_MAX) { zxc_cstream_free(cs); return 0; } const size_t bound = (size_t)bound64; if (bound > comp_cap) { void* nb = realloc(comp_buf, bound); if (!nb) { zxc_cstream_free(cs); return 0; } comp_buf = (uint8_t*)nb; comp_cap = bound; } size_t comp_len = 0; size_t in_pos = 0; /* Feed input in `in_chunk`-sized slices. */ while (in_pos < size) { const size_t feed = (size - in_pos < in_chunk) ? (size - in_pos) : in_chunk; zxc_inbuf_t in = {.src = data + in_pos, .size = feed, .pos = 0}; /* Drain repeatedly until this slice is fully consumed. */ for (;;) { size_t window = bound - comp_len; if (window > FUZZ_PSTREAM_OUT_CAP) window = FUZZ_PSTREAM_OUT_CAP; if (window == 0) { /* Bound exhausted - shouldn't happen with zxc_compress_bound. */ zxc_cstream_free(cs); return 0; } zxc_outbuf_t ob = {.dst = comp_buf + comp_len, .size = window, .pos = 0}; const int64_t r = zxc_cstream_compress(cs, &ob, &in); if (r < 0) { zxc_cstream_free(cs); return 0; } comp_len += ob.pos; if (in.pos == in.size && r == 0) break; } in_pos += in.pos; } /* Finalize: zxc_cstream_end may report >0 pending bytes; drain until 0. */ for (;;) { size_t window = bound - comp_len; if (window > FUZZ_PSTREAM_OUT_CAP) window = FUZZ_PSTREAM_OUT_CAP; if (window == 0) { zxc_cstream_free(cs); return 0; } zxc_outbuf_t ob = {.dst = comp_buf + comp_len, .size = window, .pos = 0}; const int64_t pending = zxc_cstream_end(cs, &ob); if (pending < 0) { zxc_cstream_free(cs); return 0; } comp_len += ob.pos; if (pending == 0) break; } zxc_cstream_free(cs); /* ------------------------------------------------------------------ */ /* Phase 2: Push decompression with chunked feeding + roundtrip */ /* ------------------------------------------------------------------ */ if (size > round_cap) { void* nb = realloc(round_buf, size); if (!nb) return 0; round_buf = (uint8_t*)nb; round_cap = size; } const int64_t dsize = drip_decompress(comp_buf, comp_len, round_buf, size, in_chunk, checksum_enabled); if (dsize >= 0) { assert((size_t)dsize == size); assert(memcmp(data, round_buf, size) == 0); } /* ------------------------------------------------------------------ */ /* Phase 3: Parser fuzzing - feed raw input directly */ /* */ /* This is where malformed-frame and truncation bugs live. We don't */ /* care about the result, only that no UB / crash occurs. */ /* ------------------------------------------------------------------ */ if (raw_size > 0) { /* Output is discarded - a single small stack buffer suffices and * keeps the parser's resumption code (small ob.size) under fuzz. */ uint8_t scratch[FUZZ_PSTREAM_OUT_CAP]; const size_t parser_chunk = derive_chunk_size(raw_data[raw_size - 1]); zxc_decompress_opts_t opts = {.checksum_enabled = checksum_enabled}; zxc_dstream* ds = zxc_dstream_create(&opts); if (ds) { size_t pos = 0; int rounds = 0; while (pos < raw_size && rounds < 1024) { const size_t feed = (raw_size - pos < parser_chunk) ? (raw_size - pos) : parser_chunk; zxc_inbuf_t in = {.src = raw_data + pos, .size = feed, .pos = 0}; zxc_outbuf_t ob = {.dst = scratch, .size = sizeof scratch, .pos = 0}; const size_t before_in = in.pos; const size_t before_out = ob.pos; const int64_t r = zxc_dstream_decompress(ds, &ob, &in); pos += in.pos; if (r < 0) break; if (in.pos == before_in && ob.pos == before_out) break; rounds++; } zxc_dstream_free(ds); } } return 0; } zxc-0.11.0/tests/fuzz_roundtrip.c000066400000000000000000000030511520102567100170050ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ #include #include #include #include #include #include "../include/zxc_buffer.h" #define FUZZ_ROUNDTRIP_MAX_INPUT (4 << 20) /* 4 MiB */ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { static void* comp_buf = NULL; static size_t comp_cap = 0; static void* decomp_buf = NULL; static size_t decomp_cap = 0; if (size > FUZZ_ROUNDTRIP_MAX_INPUT) return 0; const uint64_t bound64 = zxc_compress_bound(size); if (bound64 == 0 || bound64 > SIZE_MAX) return 0; const size_t bound = (size_t)bound64; if (bound > comp_cap) { void* new_buf = realloc(comp_buf, bound); if (!new_buf) return 0; comp_buf = new_buf; comp_cap = bound; } const int level = size > 0 ? (data[0] % 6) + 1 : 1; zxc_compress_opts_t copts = {.level = level}; const int64_t csize = zxc_compress(data, size, comp_buf, bound, &copts); if (csize < 0) return 0; if (size == 0) return 0; if (size > decomp_cap) { void* new_buf = realloc(decomp_buf, size); if (!new_buf) return 0; decomp_buf = new_buf; decomp_cap = size; } const int64_t dsize = zxc_decompress(comp_buf, (size_t)csize, decomp_buf, size, NULL); if (dsize >= 0) { assert((size_t)dsize == size); assert(memcmp(data, decomp_buf, size) == 0); } return 0; }zxc-0.11.0/tests/fuzz_seekable.c000066400000000000000000000144601520102567100165400ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /** * @file fuzz_seekable.c * @brief Fuzzer for the seekable random-access decompression API. * * Strategy: compress fuzzed input with seekable=1, then exercise the full * seekable read path (open, metadata getters, single-threaded decompress, * multi-threaded decompress) and verify data integrity. */ #include #include #include #include #include #include "../include/zxc_buffer.h" #include "../include/zxc_seekable.h" #define FUZZ_SEEKABLE_MAX_INPUT (4 << 20) /* 4 MiB */ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { if (size < 2) return 0; /* Save original input for phase 7 (raw parser fuzzing) */ const uint8_t* const raw_data = data; const size_t raw_size = size; /* Use first byte as level, second as flags */ const int level = (data[0] % 5) + 1; const int use_checksum = data[1] & 1; const int use_mt = data[1] & 2; data += 2; size -= 2; if (size == 0 || size > FUZZ_SEEKABLE_MAX_INPUT) return 0; /* Persistent buffers - reused across iterations to reduce allocator pressure */ static uint8_t* comp_buf = NULL; static size_t comp_cap = 0; static uint8_t* decomp_buf = NULL; static size_t decomp_cap = 0; /* ------------------------------------------------------------------ */ /* Phase 1: Compress with seekable=1 */ /* ------------------------------------------------------------------ */ const uint64_t bound64 = zxc_compress_bound(size); if (bound64 == 0 || bound64 > SIZE_MAX) return 0; const size_t bound = (size_t)bound64; if (bound > comp_cap) { void* new_buf = realloc(comp_buf, bound); if (!new_buf) return 0; comp_buf = (uint8_t*)new_buf; comp_cap = bound; } zxc_compress_opts_t copts = { .level = level, .checksum_enabled = use_checksum, .seekable = 1, }; const int64_t csize = zxc_compress(data, size, comp_buf, bound, &copts); if (csize < 0) return 0; /* ------------------------------------------------------------------ */ /* Phase 2: Open seekable handle */ /* ------------------------------------------------------------------ */ zxc_seekable* s = zxc_seekable_open(comp_buf, (size_t)csize); if (!s) return 0; /* ------------------------------------------------------------------ */ /* Phase 3: Exercise metadata getters */ /* ------------------------------------------------------------------ */ const uint32_t num_blocks = zxc_seekable_get_num_blocks(s); const uint64_t total_decomp = zxc_seekable_get_decompressed_size(s); assert(total_decomp == size); for (uint32_t i = 0; i < num_blocks; i++) { const uint32_t csz = zxc_seekable_get_block_comp_size(s, i); const uint32_t dsz = zxc_seekable_get_block_decomp_size(s, i); assert(csz > 0); assert(dsz > 0); (void)csz; (void)dsz; } /* Out-of-range access should return 0 */ assert(zxc_seekable_get_block_comp_size(s, num_blocks) == 0); assert(zxc_seekable_get_block_decomp_size(s, num_blocks) == 0); /* ------------------------------------------------------------------ */ /* Phase 4: Full decompression via seekable range */ /* ------------------------------------------------------------------ */ if (size > decomp_cap) { void* new_buf = realloc(decomp_buf, size); if (!new_buf) { zxc_seekable_free(s); return 0; } decomp_buf = (uint8_t*)new_buf; decomp_cap = size; } int64_t dec_result; if (use_mt && size > 4096) { dec_result = zxc_seekable_decompress_range_mt(s, decomp_buf, size, 0, size, 2); } else { dec_result = zxc_seekable_decompress_range(s, decomp_buf, size, 0, size); } assert(dec_result == (int64_t)size); assert(memcmp(data, decomp_buf, size) == 0); /* ------------------------------------------------------------------ */ /* Phase 5: Partial range decompression (sub-block extraction) */ /* ------------------------------------------------------------------ */ if (size >= 4) { /* Extract a range from the middle */ const size_t off = size / 4; const size_t len = size / 2; dec_result = zxc_seekable_decompress_range(s, decomp_buf, len, off, len); assert(dec_result == (int64_t)len); assert(memcmp(data + off, decomp_buf, len) == 0); } /* ------------------------------------------------------------------ */ /* Phase 6: Edge cases */ /* ------------------------------------------------------------------ */ /* Zero-length read */ dec_result = zxc_seekable_decompress_range(s, decomp_buf, size, 0, 0); assert(dec_result == 0); /* Out-of-bounds read */ dec_result = zxc_seekable_decompress_range(s, decomp_buf, size, total_decomp, 1); assert(dec_result < 0); /* NULL handle */ assert(zxc_seekable_get_num_blocks(NULL) == 0); assert(zxc_seekable_get_decompressed_size(NULL) == 0); /* ------------------------------------------------------------------ */ /* Cleanup */ /* ------------------------------------------------------------------ */ zxc_seekable_free(s); /* ------------------------------------------------------------------ */ /* Phase 7: Fuzz the parser with raw data (malformed seekable archives) */ /* ------------------------------------------------------------------ */ zxc_seekable* s2 = zxc_seekable_open(raw_data, raw_size); if (s2) { /* If it parsed, try a read - should not crash */ const uint64_t td = zxc_seekable_get_decompressed_size(s2); if (td > 0 && td <= FUZZ_SEEKABLE_MAX_INPUT) { uint8_t* tmp = (uint8_t*)malloc((size_t)td); if (tmp) { zxc_seekable_decompress_range(s2, tmp, (size_t)td, 0, (size_t)td); free(tmp); } } zxc_seekable_free(s2); } return 0; } zxc-0.11.0/tests/test_block_api.c000066400000000000000000000576071520102567100167030ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ #include "test_common.h" int test_block_api() { printf("=== TEST: Unit - Block API (zxc_compress_block/zxc_decompress_block) ===\n"); const size_t src_size = 128 * 1024; // 128 KB int result = 0; /* All resources initialized to NULL for centralized cleanup. */ uint8_t* src = NULL; uint8_t* compressed = NULL; uint8_t* decompressed = NULL; zxc_cctx* cctx = NULL; zxc_dctx* dctx = NULL; src = malloc(src_size); if (!src) goto cleanup; gen_lz_data(src, src_size); // 1. zxc_compress_block_bound const uint64_t block_bound = zxc_compress_block_bound(src_size); const uint64_t file_bound = zxc_compress_bound(src_size); if (block_bound == 0 || block_bound >= file_bound) { printf("Failed: block_bound=%llu should be >0 and < file_bound=%llu\n", (unsigned long long)block_bound, (unsigned long long)file_bound); goto cleanup; } printf(" [PASS] block_bound=%llu < file_bound=%llu\n", (unsigned long long)block_bound, (unsigned long long)file_bound); // 2. Allocate buffers and contexts compressed = malloc((size_t)block_bound); decompressed = malloc(src_size); cctx = zxc_create_cctx(NULL); dctx = zxc_create_dctx(); if (!compressed || !decompressed || !cctx || !dctx) { printf("Failed: Allocation failed\n"); goto cleanup; } // 3. Compress block (no checksum) zxc_compress_opts_t copts = {.level = 3, .checksum_enabled = 0}; int64_t csize = zxc_compress_block(cctx, src, src_size, compressed, (size_t)block_bound, &copts); if (csize <= 0) { printf("Failed: zxc_compress_block returned %lld\n", (long long)csize); goto cleanup; } printf(" [PASS] Compress block: %zu -> %lld bytes\n", src_size, (long long)csize); // 4. Decompress block (no checksum) zxc_decompress_opts_t dopts = {.checksum_enabled = 0}; int64_t dsize = zxc_decompress_block(dctx, compressed, (size_t)csize, decompressed, src_size, &dopts); if (dsize != (int64_t)src_size) { printf("Failed: zxc_decompress_block returned %lld, expected %zu\n", (long long)dsize, src_size); goto cleanup; } if (memcmp(src, decompressed, src_size) != 0) { printf("Failed: Content mismatch after block decompression\n"); goto cleanup; } printf(" [PASS] Decompress block: roundtrip OK (no checksum)\n"); // 5. With checksum enabled copts.checksum_enabled = 1; csize = zxc_compress_block(cctx, src, src_size, compressed, (size_t)block_bound, &copts); if (csize <= 0) { printf("Failed: zxc_compress_block with checksum returned %lld\n", (long long)csize); goto cleanup; } dopts.checksum_enabled = 1; dsize = zxc_decompress_block(dctx, compressed, (size_t)csize, decompressed, src_size, &dopts); if (dsize != (int64_t)src_size || memcmp(src, decompressed, src_size) != 0) { printf("Failed: Roundtrip with checksum failed\n"); goto cleanup; } printf(" [PASS] Decompress block: roundtrip OK (with checksum)\n"); // 6. Error cases if (zxc_compress_block(NULL, src, src_size, compressed, (size_t)block_bound, &copts) >= 0) { printf("Failed: Should fail with NULL cctx\n"); goto cleanup; } if (zxc_compress_block(cctx, NULL, src_size, compressed, (size_t)block_bound, &copts) >= 0) { printf("Failed: Should fail with NULL src\n"); goto cleanup; } if (zxc_decompress_block(NULL, compressed, (size_t)csize, decompressed, src_size, &dopts) >= 0) { printf("Failed: Should fail with NULL dctx\n"); goto cleanup; } printf(" [PASS] Error cases handled correctly\n"); // 7. Context reuse across multiple blocks for (int i = 0; i < 3; i++) { gen_lz_data(src, src_size); src[0] = (uint8_t)i; copts.checksum_enabled = 0; csize = zxc_compress_block(cctx, src, src_size, compressed, (size_t)block_bound, &copts); if (csize <= 0) { printf("Failed: Reuse iteration %d compress failed\n", i); goto cleanup; } dopts.checksum_enabled = 0; dsize = zxc_decompress_block(dctx, compressed, (size_t)csize, decompressed, src_size, &dopts); if (dsize != (int64_t)src_size || memcmp(src, decompressed, src_size) != 0) { printf("Failed: Reuse iteration %d roundtrip failed\n", i); goto cleanup; } } printf(" [PASS] Context reuse: 3 independent blocks OK\n"); // 8. Auto-resize: src_size > block_size must succeed (ZXC auto-sizes) { zxc_compress_opts_t guard_opts = {.level = 3, .block_size = 4096, .checksum_enabled = 0}; int64_t guard_rc = zxc_compress_block(cctx, src, src_size, compressed, (size_t)block_bound, &guard_opts); if (guard_rc <= 0) { printf("Failed: src_size > block_size should auto-resize, got %lld\n", (long long)guard_rc); goto cleanup; } printf(" [PASS] Auto-resize: src_size > block_size succeeded\n"); } printf("PASS\n\n"); result = 1; cleanup: zxc_free_cctx(cctx); /* safe with NULL */ zxc_free_dctx(dctx); /* safe with NULL */ free(compressed); free(decompressed); free(src); return result; } /* Roundtrip helper: compress src into a newly-malloc'd buffer; return compressed size (or <0). */ static int64_t sbs_compress(const uint8_t* src, size_t src_size, int level, int checksum, uint8_t** out_buf, size_t* out_cap) { const uint64_t cbound = zxc_compress_block_bound(src_size); uint8_t* buf = (uint8_t*)malloc((size_t)cbound); if (!buf) return -1; zxc_cctx* cctx = zxc_create_cctx(NULL); zxc_compress_opts_t co = {.level = level, .checksum_enabled = checksum, .block_size = src_size}; int64_t csz = zxc_compress_block(cctx, src, src_size, buf, (size_t)cbound, &co); zxc_free_cctx(cctx); if (csz <= 0) { free(buf); return csz; } *out_buf = buf; *out_cap = (size_t)cbound; return csz; } int test_decompress_block_safe() { printf("=== TEST: Unit - zxc_decompress_block_safe ===\n"); const size_t sizes[] = {4 * 1024, 64 * 1024, 256 * 1024, 2 * 1024 * 1024}; const int levels[] = {1, 3, 5, 6}; /* 1. Roundtrip with dst_capacity == uncompressed_size at multiple sizes & levels. */ for (size_t si = 0; si < sizeof(sizes) / sizeof(sizes[0]); si++) { for (size_t li = 0; li < sizeof(levels) / sizeof(levels[0]); li++) { for (int checksum = 0; checksum <= 1; checksum++) { const size_t n = sizes[si]; const int lvl = levels[li]; uint8_t* src = (uint8_t*)malloc(n); if (!src) { printf("Failed: malloc src\n"); return 0; } gen_lz_data(src, n); uint8_t* comp = NULL; size_t comp_cap = 0; int64_t csz = sbs_compress(src, n, lvl, checksum, &comp, &comp_cap); if (csz <= 0) { printf("Failed: compress (n=%zu lvl=%d chk=%d) -> %lld\n", n, lvl, checksum, (long long)csz); free(src); free(comp); return 0; } uint8_t* dst = (uint8_t*)malloc(n); /* tight: no tail pad */ if (!dst) { free(src); free(comp); return 0; } zxc_dctx* dctx = zxc_create_dctx(); zxc_decompress_opts_t dopts = {.checksum_enabled = checksum}; int64_t dsz = zxc_decompress_block_safe(dctx, comp, (size_t)csz, dst, n, &dopts); int ok = (dsz == (int64_t)n) && memcmp(src, dst, n) == 0; zxc_free_dctx(dctx); free(dst); free(comp); free(src); if (!ok) { printf("Failed: safe roundtrip n=%zu lvl=%d chk=%d -> dsz=%lld\n", n, lvl, checksum, (long long)dsz); return 0; } } } } printf(" [PASS] safe roundtrip across sizes/levels/checksum\n"); /* 2. Bit-identical vs zxc_decompress_block on the same compressed payload. */ { const size_t n = 128 * 1024; uint8_t* src = (uint8_t*)malloc(n); gen_lz_data(src, n); uint8_t* comp = NULL; size_t comp_cap = 0; int64_t csz = sbs_compress(src, n, 3, 0, &comp, &comp_cap); const uint64_t dbound = zxc_decompress_block_bound(n); uint8_t* dst_fast = (uint8_t*)malloc((size_t)dbound); uint8_t* dst_safe = (uint8_t*)malloc(n); zxc_dctx* dctx1 = zxc_create_dctx(); zxc_dctx* dctx2 = zxc_create_dctx(); int64_t r1 = zxc_decompress_block(dctx1, comp, (size_t)csz, dst_fast, (size_t)dbound, NULL); int64_t r2 = zxc_decompress_block_safe(dctx2, comp, (size_t)csz, dst_safe, n, NULL); int ok = (r1 == (int64_t)n) && (r2 == (int64_t)n) && memcmp(dst_fast, dst_safe, n) == 0; zxc_free_dctx(dctx1); zxc_free_dctx(dctx2); free(dst_safe); free(dst_fast); free(comp); free(src); if (!ok) { printf("Failed: safe/fast not bit-identical: r1=%lld r2=%lld\n", (long long)r1, (long long)r2); return 0; } printf(" [PASS] bit-identical output vs fast path\n"); } /* 3. dst_capacity < uncompressed_size -> negative error. */ { const size_t n = 64 * 1024; uint8_t* src = (uint8_t*)malloc(n); gen_lz_data(src, n); uint8_t* comp = NULL; size_t comp_cap = 0; int64_t csz = sbs_compress(src, n, 3, 0, &comp, &comp_cap); uint8_t* dst = (uint8_t*)malloc(n); zxc_dctx* dctx = zxc_create_dctx(); int64_t r = zxc_decompress_block_safe(dctx, comp, (size_t)csz, dst, n - 128, NULL); int ok = (r < 0); zxc_free_dctx(dctx); free(dst); free(comp); free(src); if (!ok) { printf("Failed: expected negative error for undersized dst, got %lld\n", (long long)r); return 0; } printf(" [PASS] negative error on dst_capacity < uncompressed_size\n"); } /* 4. Literal-heavy input: would trip OVERFLOW in the fast path when * dst_capacity == uncompressed_size; must succeed with the safe API. */ { const size_t n = 32 * 1024; uint8_t* src = (uint8_t*)malloc(n); gen_random_data(src, n); /* random data -> heavy literal runs, varint-prone */ uint8_t* comp = NULL; size_t comp_cap = 0; int64_t csz = sbs_compress(src, n, 3, 0, &comp, &comp_cap); if (csz <= 0) { printf("Failed: compress literal-heavy\n"); return 0; } uint8_t* dst = (uint8_t*)malloc(n); zxc_dctx* dctx = zxc_create_dctx(); int64_t r = zxc_decompress_block_safe(dctx, comp, (size_t)csz, dst, n, NULL); int ok = (r == (int64_t)n) && memcmp(src, dst, n) == 0; zxc_free_dctx(dctx); free(dst); free(comp); free(src); if (!ok) { printf("Failed: literal-heavy safe decode: r=%lld\n", (long long)r); return 0; } printf(" [PASS] literal-heavy tail decodes into tight dst\n"); } /* 5. Corrupted stream returns a negative error and does not crash. */ { const size_t n = 16 * 1024; uint8_t* src = (uint8_t*)malloc(n); gen_lz_data(src, n); uint8_t* comp = NULL; size_t comp_cap = 0; int64_t csz = sbs_compress(src, n, 3, 1, &comp, &comp_cap); /* with checksum */ /* Flip a byte in the payload to corrupt. */ comp[ZXC_BLOCK_HEADER_SIZE + (csz - ZXC_BLOCK_HEADER_SIZE) / 2] ^= 0xA5; uint8_t* dst = (uint8_t*)malloc(n); zxc_dctx* dctx = zxc_create_dctx(); zxc_decompress_opts_t opts = {.checksum_enabled = 1}; int64_t r = zxc_decompress_block_safe(dctx, comp, (size_t)csz, dst, n, &opts); int ok = (r < 0); zxc_free_dctx(dctx); free(dst); free(comp); free(src); if (!ok) { printf("Failed: corrupted stream should fail, got %lld\n", (long long)r); return 0; } printf(" [PASS] corrupted stream -> negative error (no crash)\n"); } printf("PASS\n\n"); return 1; } int test_decompress_block_bound() { printf("=== TEST: Unit - zxc_decompress_block_bound ===\n"); /* 1. Sanity: helper must return more than the input (tail pad > 0). */ { const size_t n = 4096; const uint64_t b = zxc_decompress_block_bound(n); if (b <= n) { printf("Failed: bound(%zu)=%llu must exceed input (tail pad missing)\n", n, (unsigned long long)b); return 0; } /* Pad is a fixed margin, so the delta must be constant. */ const uint64_t pad = b - n; const uint64_t b2 = zxc_decompress_block_bound(n * 4); if (b2 - n * 4 != pad) { printf("Failed: tail pad must be constant, got %llu vs %llu\n", (unsigned long long)pad, (unsigned long long)(b2 - n * 4)); return 0; } printf(" [PASS] bound(n) = n + %llu (constant tail pad)\n", (unsigned long long)pad); } /* 2. Overflow: huge input must return 0. */ { if (zxc_decompress_block_bound(SIZE_MAX) != 0) { printf("Failed: bound(SIZE_MAX) must return 0 on overflow\n"); return 0; } printf(" [PASS] bound(SIZE_MAX) -> 0 (overflow guard)\n"); } /* 3. Edge: bound(0) must still return a valid non-zero pad. */ { if (zxc_decompress_block_bound(0) == 0) { printf("Failed: bound(0) must be > 0 (tail pad always required)\n"); return 0; } printf(" [PASS] bound(0) > 0\n"); } /* 4. Functional: a roundtrip using bound-sized dst must succeed. */ { const size_t src_size = 64 * 1024; uint8_t* src = malloc(src_size); if (!src) return 0; gen_lz_data(src, src_size); const uint64_t cbound = zxc_compress_block_bound(src_size); uint8_t* compressed = malloc((size_t)cbound); const uint64_t dbound = zxc_decompress_block_bound(src_size); uint8_t* decompressed = malloc((size_t)dbound); zxc_cctx* cctx = zxc_create_cctx(NULL); zxc_dctx* dctx = zxc_create_dctx(); int ok = 0; if (compressed && decompressed && cctx && dctx) { zxc_compress_opts_t copts = {.level = 3}; int64_t csize = zxc_compress_block(cctx, src, src_size, compressed, (size_t)cbound, &copts); if (csize > 0) { int64_t dsize = zxc_decompress_block(dctx, compressed, (size_t)csize, decompressed, (size_t)dbound, NULL); ok = (dsize == (int64_t)src_size) && memcmp(src, decompressed, src_size) == 0; } } zxc_free_cctx(cctx); zxc_free_dctx(dctx); free(decompressed); free(compressed); free(src); if (!ok) { printf("Failed: roundtrip with bound-sized dst failed\n"); return 0; } printf(" [PASS] roundtrip into bound-sized dst OK\n"); } printf("PASS\n\n"); return 1; } /** * @brief Stress-test the block API with boundary sizes across all levels. * * Tests zxc_compress_block / zxc_decompress_block with input sizes carefully * chosen to land near internal buffer limits (mflimit, page boundaries). * * This test covers: * - Sizes near the LZ match-finder safety margin (12-20 bytes) * - Odd sizes that stress alignment assumptions * - Data patterns that trigger each block type encoder (GLO, GHI, NUM, RAW) * - All compression levels */ int test_block_api_boundary_sizes() { printf("=== TEST: Block API - Boundary Sizes ===\n"); /* Edge-case sizes: near mflimit (iend-12), near page boundaries, odd, * and large block sizes (128KB - 2MB) */ const size_t sizes[] = { 13, 14, 15, 16, 17, 19, 20, 23, 24, 25, /* Near mflimit margin */ 31, 32, 33, 48, 63, 64, 65, /* Cache line edges */ 100, 127, 128, 129, 255, 256, 257, /* Byte boundary edges */ 511, 512, 513, 1023, 1024, 1025, /* 1 KB edges */ 4095, 4096, 4097, /* Page boundary */ 8191, 8192, 8193, /* 2-page boundary */ 16383, 16384, 16385, /* 4-page boundary */ 65535, 65536, 65537, /* 64 KB boundary */ 128 * 1024, 128 * 1024 + 1, /* 128 KB block */ 256 * 1024, 256 * 1024 - 1, /* 256 KB block */ 512 * 1024, /* 512 KB block */ 1024 * 1024, /* 1 MB */ 2 * 1024 * 1024, /* 2 MB max block */ }; const int num_sizes = (int)(sizeof(sizes) / sizeof(sizes[0])); /* Data generators: each triggers a different encoder path */ typedef void (*gen_fn)(uint8_t*, size_t); const struct { const char* name; gen_fn gen; } patterns[] = { {"LZ (GLO/GHI)", gen_lz_data}, {"Random (RAW)", gen_random_data}, {"Numeric (NUM)", gen_num_data}, }; const int num_patterns = (int)(sizeof(patterns) / sizeof(patterns[0])); const size_t max_size = sizes[num_sizes - 1]; uint8_t* src = malloc(max_size); const uint64_t bound = zxc_compress_block_bound(max_size); uint8_t* compressed = malloc((size_t)bound); uint8_t* decompressed = malloc(max_size); zxc_cctx* cctx = zxc_create_cctx(NULL); zxc_dctx* dctx = zxc_create_dctx(); if (!src || !compressed || !decompressed || !cctx || !dctx) { printf(" [FAIL] allocation failed\n"); free(src); free(compressed); free(decompressed); zxc_free_cctx(cctx); zxc_free_dctx(dctx); return 0; } int failures = 0; for (int p = 0; p < num_patterns; p++) { /* Generate data once at max size; smaller tests use prefix */ patterns[p].gen(src, max_size); for (int lvl = 1; lvl <= 5; lvl++) { for (int s = 0; s < num_sizes; s++) { const size_t sz = sizes[s]; /* NUM encoder needs size >= 16 && multiple of 4 */ if (p == 2 && (sz < 16 || sz % 4 != 0)) continue; zxc_compress_opts_t copts = {.level = lvl, .checksum_enabled = 0}; const int64_t csize = zxc_compress_block( cctx, src, sz, compressed, (size_t)bound, &copts); if (csize <= 0) { /* RAW fallback or incompressible is OK, but actual errors are not */ if (csize < 0 && csize != ZXC_ERROR_DST_TOO_SMALL) { printf(" [FAIL] %s lvl=%d sz=%zu: compress error %lld\n", patterns[p].name, lvl, sz, (long long)csize); failures++; } continue; } zxc_decompress_opts_t dopts = {.checksum_enabled = 0}; const int64_t dsize = zxc_decompress_block( dctx, compressed, (size_t)csize, decompressed, sz, &dopts); if (dsize != (int64_t)sz) { printf(" [FAIL] %s lvl=%d sz=%zu: decompress returned %lld (expected %zu)\n", patterns[p].name, lvl, sz, (long long)dsize, sz); failures++; continue; } if (memcmp(src, decompressed, sz) != 0) { printf(" [FAIL] %s lvl=%d sz=%zu: content mismatch\n", patterns[p].name, lvl, sz); failures++; } } } } free(src); free(compressed); free(decompressed); zxc_free_cctx(cctx); zxc_free_dctx(dctx); if (failures > 0) { printf(" [FAIL] %d sub-tests failed\n", failures); return 0; } printf(" [PASS] All boundary sizes passed (%d patterns x 5 levels x %d sizes)\n", num_patterns, num_sizes); return 1; } /* * Regression test for blocks > 2 MiB. * * Before the varint extension, zxc_write_varint silently truncated LL/ML * values >= 2^21 (21 bits), corrupting the output for any block that * produced a literal run or match length exceeding 2 MiB. The encoder * now writes 4- and 5-byte varints (28/32 bits), matching the decoder * which already supported them. * * This test round-trips highly repetitive data in blocks of 3, 4 and 8 MiB. * The LZ77 path on such data produces match lengths close to block size, * reliably exercising the 4-byte varint branch. */ int test_block_api_large_block_varint() { printf("=== TEST: Block API - Varint >21 bits (blocks > 2 MiB) ===\n"); const struct { size_t size; const char* name; } cases[] = { {3 * 1024 * 1024, "3 MiB"}, {4 * 1024 * 1024, "4 MiB"}, {8 * 1024 * 1024, "8 MiB"}, }; const int num_cases = (int)(sizeof(cases) / sizeof(cases[0])); int failures = 0; for (int i = 0; i < num_cases; i++) { const size_t sz = cases[i].size; uint8_t* src = malloc(sz); const uint64_t bound = zxc_compress_block_bound(sz); uint8_t* compressed = bound ? malloc((size_t)bound) : NULL; uint8_t* decompressed = malloc(sz); zxc_cctx* cctx = zxc_create_cctx(NULL); zxc_dctx* dctx = zxc_create_dctx(); if (!src || !compressed || !decompressed || !cctx || !dctx) { printf(" [FAIL] %s: allocation failed\n", cases[i].name); failures++; goto per_case_cleanup; } /* Repetitive pattern: after the initial ~400 byte literal run, the * rest of the buffer matches, producing a match length close to sz * (well above 2^21) triggering the 4-byte varint encoding path. */ gen_lz_data(src, sz); for (int lvl = 1; lvl <= 5; lvl++) { zxc_compress_opts_t copts = {.level = lvl, .checksum_enabled = 1}; const int64_t csize = zxc_compress_block(cctx, src, sz, compressed, (size_t)bound, &copts); if (csize <= 0) { printf(" [FAIL] %s lvl=%d: compress returned %lld\n", cases[i].name, lvl, (long long)csize); failures++; continue; } zxc_decompress_opts_t dopts = {.checksum_enabled = 1}; const int64_t dsize = zxc_decompress_block(dctx, compressed, (size_t)csize, decompressed, sz, &dopts); if (dsize != (int64_t)sz) { printf(" [FAIL] %s lvl=%d: decompress returned %lld (expected %zu)\n", cases[i].name, lvl, (long long)dsize, sz); failures++; continue; } if (memcmp(src, decompressed, sz) != 0) { printf(" [FAIL] %s lvl=%d: content mismatch after roundtrip\n", cases[i].name, lvl); failures++; continue; } } if (!failures) { printf(" [PASS] %s roundtrip OK across levels 1-5\n", cases[i].name); } per_case_cleanup: free(src); free(compressed); free(decompressed); zxc_free_cctx(cctx); zxc_free_dctx(dctx); } if (failures > 0) { printf("FAILED: %d sub-tests failed\n", failures); return 0; } printf("PASS\n\n"); return 1; } zxc-0.11.0/tests/test_buffer_api.c000066400000000000000000000645101520102567100170510ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ #include "test_common.h" // Checks that the utility function calculates a sufficient size int test_max_compressed_size_logic() { printf("=== TEST: Unit - zxc_compress_bound ===\n"); // Case 1: 0 bytes (must at least contain the header) size_t sz0 = (size_t)zxc_compress_bound(0); if (sz0 == 0) { printf("Failed: Size for 0 bytes should not be 0 (headers required)\n"); return 0; } // Case 2: Small input size_t input_val = 100; size_t sz100 = (size_t)zxc_compress_bound(input_val); if (sz100 < input_val) { printf("Failed: Output buffer size (%zu) too small for input (%zu)\n", sz100, input_val); return 0; } // Case 3: Consistency (size should not decrease arbitrarily) if (zxc_compress_bound(2000) < zxc_compress_bound(1000)) { printf("Failed: Max size function is not monotonic\n"); return 0; } printf("PASS\n\n"); return 1; } // Checks the buffer-based API (zxc_compress / zxc_decompress) int test_buffer_api() { printf("=== TEST: Unit - Buffer API (zxc_compress/zxc_decompress) ===\n"); size_t src_size = 128 * 1024; uint8_t* src = malloc(src_size); gen_lz_data(src, src_size); // 1. Calculate max compressed size size_t max_dst_size = (size_t)zxc_compress_bound(src_size); uint8_t* compressed = malloc(max_dst_size); int checksum_enabled = 1; // 2. Compress zxc_compress_opts_t _co23 = {.level = 3, .checksum_enabled = checksum_enabled}; int64_t compressed_size = zxc_compress(src, src_size, compressed, max_dst_size, &_co23); if (compressed_size <= 0) { printf("Failed: zxc_compress returned %lld\n", (long long)compressed_size); free(src); free(compressed); return 0; } printf("Compressed %zu bytes to %lld bytes\n", src_size, (long long)compressed_size); // 3. Decompress uint8_t* decompressed = malloc(src_size); zxc_decompress_opts_t _do24 = {.checksum_enabled = checksum_enabled}; int64_t decompressed_size = zxc_decompress(compressed, (size_t)compressed_size, decompressed, src_size, &_do24); if (decompressed_size != (int64_t)src_size) { printf("Failed: zxc_decompress returned %lld, expected %zu\n", (long long)decompressed_size, src_size); free(src); free(compressed); free(decompressed); return 0; } // 4. Verify content if (memcmp(src, decompressed, src_size) != 0) { printf("Failed: Content mismatch after decompression\n"); free(src); free(compressed); free(decompressed); return 0; } // 5. Test error case: Destination too small size_t small_capacity = (size_t)(compressed_size / 2); zxc_compress_opts_t _co25 = {.level = 3, .checksum_enabled = checksum_enabled}; int64_t small_res = zxc_compress(src, src_size, compressed, small_capacity, &_co25); if (small_res >= 0) { printf("Failed: zxc_compress should fail with small buffer (returned %lld)\n", (long long)small_res); free(src); free(compressed); free(decompressed); return 0; } printf("PASS\n\n"); free(src); free(compressed); free(decompressed); return 1; } // Test zxc_get_decompressed_size int test_get_decompressed_size() { printf("=== TEST: Unit - zxc_get_decompressed_size ===\n"); // 1. Compress some data, then check decompressed size size_t src_size = 64 * 1024; uint8_t* src = malloc(src_size); gen_lz_data(src, src_size); size_t max_dst = (size_t)zxc_compress_bound(src_size); uint8_t* compressed = malloc(max_dst); zxc_compress_opts_t _co29 = {.level = 3, .checksum_enabled = 0}; int64_t comp_size = zxc_compress(src, src_size, compressed, max_dst, &_co29); if (comp_size <= 0) { printf("Failed: Compression returned 0\n"); free(src); free(compressed); return 0; } size_t reported = (size_t)zxc_get_decompressed_size(compressed, comp_size); if (reported != src_size) { printf("Failed: Expected %zu, got %zu\n", src_size, reported); free(src); free(compressed); return 0; } printf(" [PASS] Valid compressed data\n"); // 2. Too-small buffer if (zxc_get_decompressed_size(compressed, 4) != 0) { printf("Failed: Should return 0 for too-small buffer\n"); free(src); free(compressed); return 0; } printf(" [PASS] Too-small buffer\n"); // 3. Invalid magic word uint8_t bad_buf[64] = {0}; if (zxc_get_decompressed_size(bad_buf, sizeof(bad_buf)) != 0) { printf("Failed: Should return 0 for invalid magic\n"); free(src); free(compressed); return 0; } printf(" [PASS] Invalid magic word\n"); printf("PASS\n\n"); free(src); free(compressed); return 1; } int test_buffer_error_codes() { printf("=== TEST: Unit - Buffer API Error Codes ===\n"); /* ------------------------------------------------------------------ */ /* zxc_compress error paths */ /* ------------------------------------------------------------------ */ // 1. NULL src zxc_compress_opts_t _co30 = {.level = 3, .checksum_enabled = 0}; int64_t r = zxc_compress(NULL, 100, (void*)1, 100, &_co30); if (r != ZXC_ERROR_NULL_INPUT) { printf(" [FAIL] NULL src: expected %d, got %lld\n", ZXC_ERROR_NULL_INPUT, (long long)r); return 0; } printf(" [PASS] zxc_compress NULL src -> ZXC_ERROR_NULL_INPUT\n"); // 2. NULL dst zxc_compress_opts_t _co31 = {.level = 3, .checksum_enabled = 0}; r = zxc_compress((void*)1, 100, NULL, 100, &_co31); if (r != ZXC_ERROR_NULL_INPUT) { printf(" [FAIL] NULL dst: expected %d, got %lld\n", ZXC_ERROR_NULL_INPUT, (long long)r); return 0; } printf(" [PASS] zxc_compress NULL dst -> ZXC_ERROR_NULL_INPUT\n"); // 3. src_size == 0 uint8_t dummy[16]; zxc_compress_opts_t _co32 = {.level = 3, .checksum_enabled = 0}; r = zxc_compress(dummy, 0, dummy, sizeof(dummy), &_co32); if (r != ZXC_ERROR_NULL_INPUT) { printf(" [FAIL] src_size==0: expected %d, got %lld\n", ZXC_ERROR_NULL_INPUT, (long long)r); return 0; } printf(" [PASS] zxc_compress src_size==0 -> ZXC_ERROR_NULL_INPUT\n"); // 4. dst_capacity == 0 zxc_compress_opts_t _co33 = {.level = 3, .checksum_enabled = 0}; r = zxc_compress(dummy, sizeof(dummy), dummy, 0, &_co33); if (r != ZXC_ERROR_NULL_INPUT) { printf(" [FAIL] dst_cap==0: expected %d, got %lld\n", ZXC_ERROR_NULL_INPUT, (long long)r); return 0; } printf(" [PASS] zxc_compress dst_capacity==0 -> ZXC_ERROR_NULL_INPUT\n"); // 5. dst too small for file header (< 16 bytes) { uint8_t src[64]; uint8_t dst[8]; // Too small for file header (16 bytes) gen_lz_data(src, sizeof(src)); zxc_compress_opts_t _co34 = {.level = 3, .checksum_enabled = 0}; r = zxc_compress(src, sizeof(src), dst, sizeof(dst), &_co34); if (r >= 0) { printf(" [FAIL] dst too small for header: expected < 0, got %lld\n", (long long)r); return 0; } } printf(" [PASS] zxc_compress dst too small for header -> negative\n"); // 6. dst too small for data (fits header but not chunk) { const size_t src_sz = 4096; uint8_t* src = malloc(src_sz); const size_t small_dst = 128; uint8_t* dst = malloc(small_dst); gen_lz_data(src, src_sz); zxc_compress_opts_t _co35 = {.level = 3, .checksum_enabled = 0}; r = zxc_compress(src, src_sz, dst, small_dst, &_co35); if (r >= 0) { printf(" [FAIL] dst too small for chunk: expected < 0, got %lld\n", (long long)r); free(src); free(dst); return 0; } free(src); free(dst); } printf(" [PASS] zxc_compress dst too small for chunk -> negative\n"); // 7. dst too small for EOF + footer { // Compress first to find the exact compressed size, then retry with // just enough for the data blocks but not for the EOF + footer. const size_t src_sz = 256; uint8_t* src = malloc(src_sz); gen_lz_data(src, src_sz); const size_t full_cap = (size_t)zxc_compress_bound(src_sz); uint8_t* full_dst = malloc(full_cap); zxc_compress_opts_t _co36 = {.level = 3, .checksum_enabled = 0}; const int64_t full_sz = zxc_compress(src, src_sz, full_dst, full_cap, &_co36); if (full_sz <= 0) { printf(" [SKIP] Cannot prepare for EOF test\n"); free(src); free(full_dst); } else { // EOF header(8) + footer(12) = 20 bytes at the end. // Try with a buffer that's just a few bytes too small. const size_t tight = (size_t)full_sz - 5; uint8_t* tight_dst = malloc(tight); zxc_compress_opts_t _co37 = {.level = 3, .checksum_enabled = 0}; r = zxc_compress(src, src_sz, tight_dst, tight, &_co37); if (r >= 0) { printf(" [FAIL] dst too small for EOF+footer: expected < 0, got %lld\n", (long long)r); free(src); free(full_dst); free(tight_dst); return 0; } free(src); free(full_dst); free(tight_dst); } } printf(" [PASS] zxc_compress dst too small for EOF+footer -> negative\n"); /* ------------------------------------------------------------------ */ /* zxc_decompress error paths */ /* ------------------------------------------------------------------ */ // 8. NULL src zxc_decompress_opts_t _do38 = {.checksum_enabled = 0}; r = zxc_decompress(NULL, 100, (void*)1, 100, &_do38); if (r != ZXC_ERROR_NULL_INPUT) { printf(" [FAIL] decompress NULL src: expected %d, got %lld\n", ZXC_ERROR_NULL_INPUT, (long long)r); return 0; } printf(" [PASS] zxc_decompress NULL src -> ZXC_ERROR_NULL_INPUT\n"); // 9. NULL dst zxc_decompress_opts_t _do39 = {.checksum_enabled = 0}; r = zxc_decompress((void*)1, 100, NULL, 100, &_do39); if (r != ZXC_ERROR_NULL_INPUT) { printf(" [FAIL] decompress NULL dst: expected %d, got %lld\n", ZXC_ERROR_NULL_INPUT, (long long)r); return 0; } printf(" [PASS] zxc_decompress NULL dst -> ZXC_ERROR_NULL_INPUT\n"); // 10. src too small for file header { uint8_t tiny[4] = {0}; uint8_t out[64]; zxc_decompress_opts_t _do40 = {.checksum_enabled = 0}; r = zxc_decompress(tiny, sizeof(tiny), out, sizeof(out), &_do40); if (r != ZXC_ERROR_NULL_INPUT) { printf(" [FAIL] src too small: expected %d, got %lld\n", ZXC_ERROR_NULL_INPUT, (long long)r); return 0; } } printf(" [PASS] zxc_decompress src too small -> ZXC_ERROR_NULL_INPUT\n"); // 11. Bad file header (invalid magic) { uint8_t bad_src[64]; memset(bad_src, 0, sizeof(bad_src)); uint8_t out[64]; zxc_decompress_opts_t _do41 = {.checksum_enabled = 0}; r = zxc_decompress(bad_src, sizeof(bad_src), out, sizeof(out), &_do41); if (r != ZXC_ERROR_BAD_HEADER) { printf(" [FAIL] bad magic: expected %d, got %lld\n", ZXC_ERROR_BAD_HEADER, (long long)r); return 0; } } printf(" [PASS] zxc_decompress bad magic -> ZXC_ERROR_BAD_HEADER\n"); // Prepare a valid compressed buffer for subsequent decompress error tests const size_t test_src_sz = 1024; uint8_t* test_src = malloc(test_src_sz); gen_lz_data(test_src, test_src_sz); const size_t comp_cap = (size_t)zxc_compress_bound(test_src_sz); uint8_t* comp_buf = malloc(comp_cap); zxc_compress_opts_t _co42 = {.level = 3, .checksum_enabled = 1}; const int64_t comp_sz = zxc_compress(test_src, test_src_sz, comp_buf, comp_cap, &_co42); if (comp_sz <= 0) { printf(" [FAIL] Could not prepare compressed data\n"); free(test_src); free(comp_buf); return 0; } // 12. Corrupt block header (damage the first block header byte after file header) { uint8_t* corrupt = malloc((size_t)comp_sz); memcpy(corrupt, comp_buf, (size_t)comp_sz); // Corrupt the block type byte at offset ZXC_FILE_HEADER_SIZE corrupt[ZXC_FILE_HEADER_SIZE] = 0xFF; // Invalid block type uint8_t* out = malloc(test_src_sz); zxc_decompress_opts_t _do43 = {.checksum_enabled = 1}; r = zxc_decompress(corrupt, (size_t)comp_sz, out, test_src_sz, &_do43); if (r >= 0) { printf(" [FAIL] corrupt block header: expected < 0, got %lld\n", (long long)r); free(corrupt); free(out); free(test_src); free(comp_buf); return 0; } free(corrupt); free(out); } printf(" [PASS] zxc_decompress corrupt block header -> negative\n"); // 13. Truncated at EOF (missing footer) { // Find the EOF block: it ends with the footer(12 bytes) // Truncate so the footer is missing const size_t trunc_sz = (size_t)comp_sz - ZXC_FILE_FOOTER_SIZE + 2; // Cut most of footer uint8_t* out = malloc(test_src_sz); zxc_decompress_opts_t _do44 = {.checksum_enabled = 1}; r = zxc_decompress(comp_buf, trunc_sz, out, test_src_sz, &_do44); if (r >= 0) { printf(" [FAIL] truncated footer: expected < 0, got %lld\n", (long long)r); free(out); free(test_src); free(comp_buf); return 0; } free(out); } printf(" [PASS] zxc_decompress truncated footer -> negative\n"); // 14. Stored size mismatch (corrupt the source size in footer) { uint8_t* corrupt = malloc((size_t)comp_sz); memcpy(corrupt, comp_buf, (size_t)comp_sz); // Footer is at end: last 12 bytes = [src_size(8)] + [global_hash(4)] // Corrupt the source size field (add 1 to the first byte) const size_t footer_offset = (size_t)comp_sz - ZXC_FILE_FOOTER_SIZE; corrupt[footer_offset] ^= 0x01; // Flip a bit in the stored source size uint8_t* out = malloc(test_src_sz); zxc_decompress_opts_t _do45 = {.checksum_enabled = 1}; r = zxc_decompress(corrupt, (size_t)comp_sz, out, test_src_sz, &_do45); if (r >= 0) { printf(" [FAIL] size mismatch: expected < 0, got %lld\n", (long long)r); free(corrupt); free(out); free(test_src); free(comp_buf); return 0; } free(corrupt); free(out); } printf(" [PASS] zxc_decompress stored size mismatch -> negative\n"); // 15. Global checksum failure (corrupt the global hash in footer) { uint8_t* corrupt = malloc((size_t)comp_sz); memcpy(corrupt, comp_buf, (size_t)comp_sz); // Global hash is the last 4 bytes of the file corrupt[comp_sz - 1] ^= 0xFF; uint8_t* out = malloc(test_src_sz); zxc_decompress_opts_t _do46 = {.checksum_enabled = 1}; r = zxc_decompress(corrupt, (size_t)comp_sz, out, test_src_sz, &_do46); if (r != ZXC_ERROR_BAD_CHECKSUM) { printf(" [FAIL] bad global checksum: expected %d, got %lld\n", ZXC_ERROR_BAD_CHECKSUM, (long long)r); free(corrupt); free(out); free(test_src); free(comp_buf); return 0; } free(corrupt); free(out); } printf(" [PASS] zxc_decompress global checksum -> ZXC_ERROR_BAD_CHECKSUM\n"); // 16. dst too small for decompression { uint8_t* out = malloc(test_src_sz / 4); // Way too small zxc_decompress_opts_t _do47 = {.checksum_enabled = 0}; r = zxc_decompress(comp_buf, (size_t)comp_sz, out, test_src_sz / 4, &_do47); if (r >= 0) { printf(" [FAIL] dst too small for decompress: expected < 0, got %lld\n", (long long)r); free(out); free(test_src); free(comp_buf); return 0; } free(out); } printf(" [PASS] zxc_decompress dst too small -> negative\n"); free(test_src); free(comp_buf); printf("PASS\n\n"); return 1; } // Tests the buffer API scratch buffer (work_buf) used to safely absorb // zxc_copy32 wild-copy overshoot during decompression. int test_buffer_api_scratch_buf() { printf("=== TEST: Unit - Buffer API Scratch Buffer (work_buf) ===\n"); // 1. Small data roundtrip (177 bytes) { const size_t sz = 177; uint8_t src[177]; gen_lz_data(src, sz); const size_t comp_cap = (size_t)zxc_compress_bound(sz); uint8_t* comp = malloc(comp_cap); zxc_compress_opts_t _co58 = {.level = 3, .checksum_enabled = 0}; const int64_t comp_sz = zxc_compress(src, sz, comp, comp_cap, &_co58); if (comp_sz <= 0) { printf(" [FAIL] compress 177B\n"); free(comp); return 0; } uint8_t dec[177]; zxc_decompress_opts_t _do59 = {.checksum_enabled = 0}; const int64_t dec_sz = zxc_decompress(comp, (size_t)comp_sz, dec, sz, &_do59); if (dec_sz != (int64_t)sz || memcmp(src, dec, sz) != 0) { printf(" [FAIL] roundtrip 177B\n"); free(comp); return 0; } free(comp); printf(" [PASS] small data roundtrip (177 bytes)\n"); } // 2. Exact-fit destination (dst_capacity == decompressed size, no slack) { const size_t sz = 1024; uint8_t* src = malloc(sz); gen_lz_data(src, sz); const size_t comp_cap = (size_t)zxc_compress_bound(sz); uint8_t* comp = malloc(comp_cap); zxc_compress_opts_t _co60 = {.level = 1, .checksum_enabled = 1}; const int64_t comp_sz = zxc_compress(src, sz, comp, comp_cap, &_co60); if (comp_sz <= 0) { printf(" [FAIL] compress 1KB\n"); free(src); free(comp); return 0; } uint8_t* dec = malloc(sz); // exactly sz, no extra room zxc_decompress_opts_t _do61 = {.checksum_enabled = 1}; const int64_t dec_sz = zxc_decompress(comp, (size_t)comp_sz, dec, sz, &_do61); if (dec_sz != (int64_t)sz || memcmp(src, dec, sz) != 0) { printf(" [FAIL] exact-fit 1KB\n"); free(src); free(comp); free(dec); return 0; } free(src); free(comp); free(dec); printf(" [PASS] exact-fit destination (1KB)\n"); } // 3. Tiny data (1 byte) { const uint8_t src = 0x42; const size_t comp_cap = (size_t)zxc_compress_bound(1); uint8_t* comp = malloc(comp_cap); zxc_compress_opts_t _co62 = {.level = 1, .checksum_enabled = 0}; const int64_t comp_sz = zxc_compress(&src, 1, comp, comp_cap, &_co62); if (comp_sz <= 0) { printf(" [FAIL] compress 1B\n"); free(comp); return 0; } uint8_t dec = 0; zxc_decompress_opts_t _do63 = {.checksum_enabled = 0}; const int64_t dec_sz = zxc_decompress(comp, (size_t)comp_sz, &dec, 1, &_do63); if (dec_sz != 1 || dec != 0x42) { printf(" [FAIL] roundtrip 1B\n"); free(comp); return 0; } free(comp); printf(" [PASS] tiny data roundtrip (1 byte)\n"); } // 4. Malformed input must not crash (safe error return) { uint8_t garbage[64]; for (int i = 0; i < 64; i++) garbage[i] = (uint8_t)(i * 37); uint8_t out[256]; zxc_decompress_opts_t _do64 = {.checksum_enabled = 0}; const int64_t r = zxc_decompress(garbage, sizeof(garbage), out, sizeof(out), &_do64); if (r >= 0) { printf(" [FAIL] malformed input should return < 0\n"); return 0; } printf(" [PASS] malformed input -> error %lld (no crash)\n", (long long)r); } // 5. Destination too small { const size_t sz = 512; uint8_t* src = malloc(sz); gen_lz_data(src, sz); const size_t comp_cap = (size_t)zxc_compress_bound(sz); uint8_t* comp = malloc(comp_cap); zxc_compress_opts_t _co65 = {.level = 1, .checksum_enabled = 0}; const int64_t comp_sz = zxc_compress(src, sz, comp, comp_cap, &_co65); if (comp_sz <= 0) { printf(" [FAIL] compress 512B\n"); free(src); free(comp); return 0; } uint8_t tiny_dst[8]; zxc_decompress_opts_t _do66 = {.checksum_enabled = 0}; const int64_t r = zxc_decompress(comp, (size_t)comp_sz, tiny_dst, sizeof(tiny_dst), &_do66); if (r >= 0) { printf(" [FAIL] dst too small should return < 0\n"); free(src); free(comp); return 0; } free(src); free(comp); printf(" [PASS] zxc_decompress dst too small -> negative\n"); } printf("PASS\n\n"); return 1; } // Tests that the two decompression paths in zxc_decompress() produce // identical results: // - Fast path: rem_cap >= runtime_chunk_size + ZXC_PAD_SIZE // -> decompress directly into dst (enough padding for wild copies). // - Safe path: rem_cap < runtime_chunk_size + ZXC_PAD_SIZE // -> decompress into bounce buffer (work_buf), then memcpy exact result. // int test_decompress_fast_vs_safe_path() { printf("=== TEST: Unit - Decompress Fast Path vs Safe Path ===\n"); // Use a multi-block input: ZXC_BLOCK_SIZE_DEFAULT + extra so we get at least 2 blocks. // Block size = 256KB (ZXC_BLOCK_SIZE_DEFAULT). Second block is small. const size_t src_sz = ZXC_BLOCK_SIZE_DEFAULT + 4096; // 256KB + 4KB -> 2 blocks uint8_t* src = malloc(src_sz); if (!src) return 0; gen_lz_data(src, src_sz); const size_t comp_cap = (size_t)zxc_compress_bound(src_sz); uint8_t* comp = malloc(comp_cap); zxc_compress_opts_t _co67 = {.level = 3, .checksum_enabled = 1}; const int64_t comp_sz = zxc_compress(src, src_sz, comp, comp_cap, &_co67); if (comp_sz <= 0) { printf(" [FAIL] compression failed\n"); free(src); free(comp); return 0; } // ----- Sub-test 1: Fast path ----- // Provide a very large dst buffer so all chunks decompress directly into // dst (rem_cap >= runtime_chunk_size + ZXC_PAD_SIZE at every iteration). { const size_t big_cap = src_sz + ZXC_BLOCK_SIZE_DEFAULT; // way more than enough uint8_t* dst = malloc(big_cap); zxc_decompress_opts_t _do68 = {.checksum_enabled = 1}; const int64_t dec_sz = zxc_decompress(comp, (size_t)comp_sz, dst, big_cap, &_do68); if (dec_sz != (int64_t)src_sz) { printf(" [FAIL] fast path size: expected %zu, got %lld\n", src_sz, (long long)dec_sz); free(dst); free(src); free(comp); return 0; } if (memcmp(src, dst, src_sz) != 0) { printf(" [FAIL] fast path content mismatch\n"); free(dst); free(src); free(comp); return 0; } free(dst); printf(" [PASS] fast path (oversized dst)\n"); } // ----- Sub-test 2: Safe path (exact-fit) ----- // Provide dst_capacity == src_sz exactly. After the first 256KB block is // written, rem_cap for the second block (4KB) is exactly 4KB which is // < runtime_chunk_size (256KB) + ZXC_PAD_SIZE (32). This forces the // safe path (bounce buffer) for the second block. { uint8_t* dst = malloc(src_sz); // no slack at all zxc_decompress_opts_t _do69 = {.checksum_enabled = 1}; const int64_t dec_sz = zxc_decompress(comp, (size_t)comp_sz, dst, src_sz, &_do69); if (dec_sz != (int64_t)src_sz) { printf(" [FAIL] safe path size: expected %zu, got %lld\n", src_sz, (long long)dec_sz); free(dst); free(src); free(comp); return 0; } if (memcmp(src, dst, src_sz) != 0) { printf(" [FAIL] safe path content mismatch\n"); free(dst); free(src); free(comp); return 0; } free(dst); printf(" [PASS] safe path (exact-fit dst)\n"); } // ----- Sub-test 3: Boundary ----- // dst_capacity = src_sz + ZXC_PAD_SIZE - 1 (just below the fast path // threshold for the LAST chunk). The last chunk should still fall into // the safe path here. { const size_t tight_cap = src_sz + ZXC_PAD_SIZE - 1; uint8_t* dst = malloc(tight_cap); zxc_decompress_opts_t _do70 = {.checksum_enabled = 1}; const int64_t dec_sz = zxc_decompress(comp, (size_t)comp_sz, dst, tight_cap, &_do70); if (dec_sz != (int64_t)src_sz) { printf(" [FAIL] boundary size: expected %zu, got %lld\n", src_sz, (long long)dec_sz); free(dst); free(src); free(comp); return 0; } if (memcmp(src, dst, src_sz) != 0) { printf(" [FAIL] boundary content mismatch\n"); free(dst); free(src); free(comp); return 0; } free(dst); printf(" [PASS] boundary (dst = src_sz + PAD - 1)\n"); } // ----- Sub-test 4: Safe path with dst too small ----- // The safe path detects that the decompressed chunk doesn't fit and // returns ZXC_ERROR_DST_TOO_SMALL (covers the res > rem_cap guard). { const size_t tiny_cap = ZXC_BLOCK_SIZE_DEFAULT / 2; // Enough for half a block uint8_t* dst = malloc(tiny_cap); zxc_decompress_opts_t _do71 = {.checksum_enabled = 0}; const int64_t dec_sz = zxc_decompress(comp, (size_t)comp_sz, dst, tiny_cap, &_do71); if (dec_sz >= 0) { printf(" [FAIL] safe path dst-too-small should fail, got %lld\n", (long long)dec_sz); free(dst); free(src); free(comp); return 0; } free(dst); printf(" [PASS] safe path dst too small -> negative\n"); } free(src); free(comp); printf("PASS\n\n"); return 1; } zxc-0.11.0/tests/test_cli.sh000077500000000000000000000774441520102567100157230ustar00rootroot00000000000000#!/bin/bash # ZXC - High-performance lossless compression # # Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. # SPDX-License-Identifier: BSD-3-Clause # # Test script for ZXC CLI # Usage: ./tests/test_cli.sh [path_to_zxc_binary] set -e # Default binary path ZXC_BIN=${1:-"../build/zxc"} # Test Directory (to isolate test files) TEST_DIR="./test_tmp" mkdir -p "$TEST_DIR" # Test Files TEST_FILE="$TEST_DIR/testdata" TEST_FILE_XC="${TEST_FILE}.zxc" TEST_FILE_DEC="${TEST_FILE}.dec" PIPE_XC="$TEST_DIR/test_pipe.zxc" PIPE_DEC="$TEST_DIR/test_pipe.dec" # Variables for checking file existence TEST_FILE_XC_BASH="$TEST_FILE_XC" TEST_FILE_DEC_BASH="${TEST_FILE}.dec" # Arguments passed to ZXC (relative to test_tmp) TEST_FILE_ARG="$TEST_FILE" TEST_FILE_XC_ARG="$TEST_FILE_XC" RED='\033[0;31m' GREEN='\033[0;32m' NC='\033[0m' # No Color log_pass() { echo -e "${GREEN}[PASS]${NC} $1" } log_fail() { echo -e "${RED}[FAIL]${NC} $1" exit 1 } # cleanup on exit cleanup() { echo "Cleaning up..." rm -rf "$TEST_DIR" } trap cleanup EXIT # 0. Check binary if [ ! -f "$ZXC_BIN" ]; then log_fail "Binary not found at $ZXC_BIN. Please build the project first." fi echo "Using binary: $ZXC_BIN" # 1. Generate Test File (Lorem Ipsum) echo "Generating test file..." cat > "$TEST_FILE" <> "${TEST_FILE}.tmp" done mv "${TEST_FILE}.tmp" "$TEST_FILE" FILE_SIZE=$(wc -c < "$TEST_FILE" | tr -d ' ') echo "Test file generated: $TEST_FILE ($FILE_SIZE bytes)" # Helper: Wait for file to be ready and readable wait_for_file() { local file="$1" local retries=10 local count=0 # On Windows, file locking can cause race conditions immediately after creation. while [ $count -lt $retries ]; do if [ -f "$file" ]; then # Try to read a byte to ensure it's not exclusively locked if head -c 1 "$file" >/dev/null 2>&1; then return 0 fi fi sleep 1 count=$((count + 1)) done echo "Timeout waiting for file '$file' to be readable." ls -l "$file" 2>/dev/null || echo "File not found in ls." return 1 } # 2. Basic Round-Trip echo "Testing Basic Round-Trip..." if ! "$ZXC_BIN" -z -k "$TEST_FILE_ARG"; then log_fail "Compression command failed (exit code $?)" fi # Wait for output if ! wait_for_file "$TEST_FILE_XC_BASH"; then log_fail "Compression succeeded but output file '$TEST_FILE_XC_BASH' is not accessible." fi # Decompress to stdout echo "Attempting decompression of: $TEST_FILE_XC_ARG" if ! "$ZXC_BIN" -d -c "$TEST_FILE_XC_ARG" > "$TEST_FILE_DEC_BASH"; then echo "Decompress to stdout failed. Retrying once..." sleep 1 if ! "$ZXC_BIN" -d -c "$TEST_FILE_XC_ARG" > "$TEST_FILE_DEC_BASH"; then log_fail "Decompression to stdout failed." fi fi # Decompress to file rm -f "$TEST_FILE" sleep 1 if ! "$ZXC_BIN" -d -k "$TEST_FILE_XC_ARG"; then echo "Decompress to file failed. Retrying once..." sleep 1 if ! "$ZXC_BIN" -d -k "$TEST_FILE_XC_ARG"; then log_fail "Decompression to file failed." fi fi if ! wait_for_file "$TEST_FILE"; then log_fail "Decompression failed to recreate original file." fi mv "$TEST_FILE" "$TEST_FILE_DEC_BASH" # Re-generate source for valid comparison cat > "$TEST_FILE" <> "${TEST_FILE}.tmp" done mv "${TEST_FILE}.tmp" "$TEST_FILE" if cmp -s "$TEST_FILE" "$TEST_FILE_DEC_BASH"; then log_pass "Basic Round-Trip" else log_fail "Basic Round-Trip content mismatch" fi # 3. Piping echo "Testing Piping..." rm -f "$PIPE_XC" "$PIPE_DEC" cat "$TEST_FILE" | "$ZXC_BIN" > "$PIPE_XC" if [ ! -s "$PIPE_XC" ]; then log_fail "Piping compression failed (empty output)" fi cat "$PIPE_XC" | "$ZXC_BIN" -d > "$PIPE_DEC" if [ ! -s "$PIPE_DEC" ]; then log_fail "Piping decompression failed (empty output)" fi if cmp -s "$TEST_FILE" "$PIPE_DEC"; then log_pass "Piping" else log_fail "Piping content mismatch" fi # 4. Flags echo "Testing Flags..." # Level "$ZXC_BIN" -1 -k -f "$TEST_FILE_ARG" if [ ! -f "$TEST_FILE_XC_BASH" ]; then log_fail "Level 1 flag failed"; fi log_pass "Flag -1" # Force Overwrite (-f) touch "$TEST_DIR/out.zxc" touch "${TEST_FILE_XC_BASH}" set +e "$ZXC_BIN" -z -k "$TEST_FILE_ARG" > /dev/null 2>&1 RET=$? set -e if [ $RET -eq 0 ]; then log_fail "Should have failed to overwrite without -f" else log_pass "Overwrite protection verified" fi "$ZXC_BIN" -z -k -f "$TEST_FILE_ARG" if [ $? -eq 0 ]; then log_pass "Force overwrite (-f)" else log_fail "Force overwrite failed" fi # 5. Benchmark echo "Testing Benchmark..." "$ZXC_BIN" -b1 "$TEST_FILE_ARG" > /dev/null if [ $? -eq 0 ]; then log_pass "Benchmark mode" else log_fail "Benchmark mode failed" fi # 6. Error Handling echo "Testing Error Handling..." set +e "$ZXC_BIN" "nonexistent_file" > /dev/null 2>&1 RET=$? set -e if [ $RET -ne 0 ]; then log_pass "Missing file error handled" else log_fail "Missing file should return error" fi # 7. Version echo "Testing Version..." OUT_VER=$("$ZXC_BIN" -V) if [[ "$OUT_VER" == *"zxc"* ]]; then log_pass "Version flag" else log_fail "Version flag failed" fi # 8. Checksum echo "Testing Checksum..." "$ZXC_BIN" -C -k -f "$TEST_FILE_ARG" if [ ! -f "$TEST_FILE_XC_BASH" ]; then log_fail "Checksum compression failed"; fi rm -f "$TEST_FILE" "$ZXC_BIN" -d "$TEST_FILE_XC_ARG" if [ ! -f "$TEST_FILE" ]; then log_fail "Checksum decompression failed"; fi log_pass "Checksum enabled (-C)" "$ZXC_BIN" -N -k -f "$TEST_FILE_ARG" if [ ! -f "$TEST_FILE_XC_BASH" ]; then log_fail "No-Checksum compression failed"; fi rm -f "$TEST_FILE" "$ZXC_BIN" -d "$TEST_FILE_XC_ARG" if [ ! -f "$TEST_FILE" ]; then log_fail "No-Checksum decompression failed"; fi log_pass "Checksum disabled (-N)" # 9. Integrity Test (-t) echo "Testing Integrity Check (-t)..." "$ZXC_BIN" -z -k -f -C "$TEST_FILE_ARG" # Valid file should pass and show "OK" OUT=$("$ZXC_BIN" -t "$TEST_FILE_XC_ARG") if [[ "$OUT" == *": OK"* ]]; then log_pass "Integrity check passed on valid file" else log_fail "Integrity check failed on valid file (expected OK output)" fi # Verbose test mode OUT=$("$ZXC_BIN" -t -v "$TEST_FILE_XC_ARG") if [[ "$OUT" == *": OK"* ]] && [[ "$OUT" == *"Checksum:"* ]]; then log_pass "Integrity check verbose mode" else log_fail "Integrity check verbose mode failed" fi # Corrupt file should fail and show "FAILED" # Corrupt a byte in the middle of the file (after header) printf '\xff' | dd of="$TEST_FILE_XC_ARG" bs=1 seek=100 count=1 conv=notrunc 2>/dev/null set +e OUT=$("$ZXC_BIN" -t "$TEST_FILE_XC_ARG" 2>&1) RET=$? set -e if [ $RET -ne 0 ] && [[ "$OUT" == *": FAILED"* ]]; then log_pass "Integrity check correctly failed on corrupt file" else log_fail "Integrity check PASSED on corrupt file (False Negative)" fi # 10. Global Checksum Integrity echo "Testing Global Checksum Integrity..." "$ZXC_BIN" -z -k -f -C "$TEST_FILE_ARG" # Corrupt the last byte (part of Global Checksum) FILE_SZ=$(wc -c < "$TEST_FILE_XC_ARG" | tr -d ' ') LAST_BYTE_OFFSET=$((FILE_SZ - 1)) printf '\x00' | dd of="$TEST_FILE_XC_ARG" bs=1 seek=$LAST_BYTE_OFFSET count=1 conv=notrunc 2>/dev/null set +e OUT=$("$ZXC_BIN" -t "$TEST_FILE_XC_ARG" 2>&1) RET=$? set -e if [ $RET -ne 0 ] && [[ "$OUT" == *": FAILED"* ]]; then log_pass "Integrity check correctly failed on corrupt Global Checksum" else log_fail "Integrity check PASSED on corrupt Global Checksum (False Negative)" fi # Ensure no output file is created if [ -f "${TEST_FILE}.zxc.zxc" ] || [ -f "${TEST_FILE}.zxc.dec" ]; then log_fail "Integrity check created output file unexpectedly" fi # 11. List Command (-l) echo "Testing List Command (-l)..." "$ZXC_BIN" -z -k -f -C "$TEST_FILE_ARG" # Normal list mode OUT=$("$ZXC_BIN" -l "$TEST_FILE_XC_ARG") if [[ "$OUT" == *"Compressed"* ]] && [[ "$OUT" == *"Uncompressed"* ]] && [[ "$OUT" == *"Ratio"* ]]; then log_pass "List command basic output" else log_fail "List command basic output failed" fi # Verbose list mode OUT=$("$ZXC_BIN" -l -v "$TEST_FILE_XC_ARG") if [[ "$OUT" == *"Block Format:"* ]] && [[ "$OUT" == *"Block Units:"* ]] && [[ "$OUT" == *"Checksum Method:"* ]]; then log_pass "List command verbose output" else log_fail "List command verbose output failed" fi # List with invalid file should fail set +e "$ZXC_BIN" -l "nonexistent_file" > /dev/null 2>&1 RET=$? set -e if [ $RET -ne 0 ]; then log_pass "List command error handling" else log_fail "List command should fail on nonexistent file" fi # 12. Compression Levels (All) echo "Testing All Compression Levels..." for LEVEL in 1 2 3 4 5 6; do "$ZXC_BIN" -$LEVEL -c -k "$TEST_FILE_ARG" > "$TEST_DIR/test_lvl${LEVEL}.zxc" if [ ! -f "$TEST_DIR/test_lvl${LEVEL}.zxc" ]; then log_fail "Compression level $LEVEL failed" fi # Decompress and verify "$ZXC_BIN" -d -c "$TEST_DIR/test_lvl${LEVEL}.zxc" > "$TEST_DIR/test_lvl${LEVEL}.dec" if ! cmp -s "$TEST_FILE" "$TEST_DIR/test_lvl${LEVEL}.dec"; then log_fail "Level $LEVEL decompression mismatch" fi SIZE=$(wc -c < "$TEST_DIR/test_lvl${LEVEL}.zxc" | tr -d ' ') log_pass "Level $LEVEL (Size: $SIZE bytes)" done # 13. Data Type Tests echo "Testing Different Data Types..." # 13.1 Highly Repetitive Text (Best Compression) echo " Testing repetitive data..." yes "Lorem ipsum dolor sit amet" | head -n 1000 > "$TEST_DIR/test_repetitive.txt" "$ZXC_BIN" -3 -k "$TEST_DIR/test_repetitive.txt" if [ ! -f "$TEST_DIR/test_repetitive.txt.zxc" ]; then log_fail "Repetitive data compression failed" fi "$ZXC_BIN" -d -c "$TEST_DIR/test_repetitive.txt.zxc" > "$TEST_DIR/test_repetitive.dec" if cmp -s "$TEST_DIR/test_repetitive.txt" "$TEST_DIR/test_repetitive.dec"; then SIZE_ORIG=$(wc -c < "$TEST_DIR/test_repetitive.txt" | tr -d ' ') SIZE_COMP=$(wc -c < "$TEST_DIR/test_repetitive.txt.zxc" | tr -d ' ') RATIO=$((SIZE_ORIG / SIZE_COMP)) log_pass "Repetitive text (Ratio: ${RATIO}:1)" else log_fail "Repetitive data round-trip failed" fi # 13.2 Binary Zeros (Highly Compressible) echo " Testing binary zeros..." dd if=/dev/zero bs=1024 count=100 of="$TEST_DIR/test_zeros.bin" 2>/dev/null "$ZXC_BIN" -3 -k "$TEST_DIR/test_zeros.bin" if [ ! -f "$TEST_DIR/test_zeros.bin.zxc" ]; then log_fail "Binary zeros compression failed" fi "$ZXC_BIN" -d -c "$TEST_DIR/test_zeros.bin.zxc" > "$TEST_DIR/test_zeros.dec" if cmp -s "$TEST_DIR/test_zeros.bin" "$TEST_DIR/test_zeros.dec"; then SIZE_ORIG=$(wc -c < "$TEST_DIR/test_zeros.bin" | tr -d ' ') SIZE_COMP=$(wc -c < "$TEST_DIR/test_zeros.bin.zxc" | tr -d ' ') RATIO=$((SIZE_ORIG / SIZE_COMP)) log_pass "Binary zeros (Ratio: ${RATIO}:1)" else log_fail "Binary zeros round-trip failed" fi # 13.3 Random Data (Incompressible - Should use RAW blocks) echo " Testing random data (incompressible)..." dd if=/dev/urandom bs=1024 count=100 of="$TEST_DIR/test_random.bin" 2>/dev/null "$ZXC_BIN" -3 -k "$TEST_DIR/test_random.bin" if [ ! -f "$TEST_DIR/test_random.bin.zxc" ]; then log_fail "Random data compression failed" fi "$ZXC_BIN" -d -c "$TEST_DIR/test_random.bin.zxc" > "$TEST_DIR/test_random.dec" if cmp -s "$TEST_DIR/test_random.bin" "$TEST_DIR/test_random.dec"; then SIZE_ORIG=$(wc -c < "$TEST_DIR/test_random.bin" | tr -d ' ') SIZE_COMP=$(wc -c < "$TEST_DIR/test_random.bin.zxc" | tr -d ' ') # Random data should expand slightly (RAW blocks + headers) if [ $SIZE_COMP -le $((SIZE_ORIG + 200)) ]; then log_pass "Random data (RAW blocks, minimal expansion)" else log_fail "Random data expanded too much" fi else log_fail "Random data round-trip failed" fi # 14. Large File Test (Performance Validation) echo "Testing Large Files..." dd if=/dev/zero bs=1M count=10 of="$TEST_DIR/test_large.bin" 2>/dev/null if ! "$ZXC_BIN" -3 -k "$TEST_DIR/test_large.bin"; then log_fail "Large file compression failed" fi if ! "$ZXC_BIN" -d -c "$TEST_DIR/test_large.bin.zxc" > "$TEST_DIR/test_large.dec"; then log_fail "Large file decompression failed" fi if cmp -s "$TEST_DIR/test_large.bin" "$TEST_DIR/test_large.dec"; then log_pass "Large file (10 MB) round-trip" else log_fail "Large file content mismatch" fi # 15. One-Pass Pipe Round-Trip (Critical for Streaming) echo "Testing One-Pass Pipe Round-Trip..." cat "$TEST_FILE" | "$ZXC_BIN" -c | "$ZXC_BIN" -dc > "$TEST_DIR/test_pipe_onepass.dec" if cmp -s "$TEST_FILE" "$TEST_DIR/test_pipe_onepass.dec"; then log_pass "One-pass pipe round-trip" else log_fail "One-pass pipe round-trip content mismatch" fi # 16. Empty File Edge Case echo "Testing Empty File..." touch "$TEST_DIR/test_empty.txt" "$ZXC_BIN" -3 -k "$TEST_DIR/test_empty.txt" if [ ! -f "$TEST_DIR/test_empty.txt.zxc" ]; then log_fail "Empty file compression failed" fi "$ZXC_BIN" -d -c "$TEST_DIR/test_empty.txt.zxc" > "$TEST_DIR/test_empty.dec" if [ ! -s "$TEST_DIR/test_empty.dec" ]; then log_pass "Empty file round-trip" else log_fail "Empty file produced non-empty output" fi # 17. Stdin Detection (No -c flag needed for compression) echo "Testing Stdin Auto-Detection..." cat "$TEST_FILE" | "$ZXC_BIN" > "$TEST_DIR/test_stdin_auto.zxc" 2>/dev/null if [ -s "$TEST_DIR/test_stdin_auto.zxc" ]; then "$ZXC_BIN" -d -c "$TEST_DIR/test_stdin_auto.zxc" > "$TEST_DIR/test_stdin_auto.dec" if cmp -s "$TEST_FILE" "$TEST_DIR/test_stdin_auto.dec"; then log_pass "Stdin auto-detection (compression)" else log_fail "Stdin auto-detection content mismatch" fi else log_fail "Stdin auto-detection failed (empty output)" fi # 18. Keep Source File (-k) echo "Testing Keep Source (-k)..." cp "$TEST_FILE" "$TEST_DIR/test_keep.txt" "$ZXC_BIN" -k "$TEST_DIR/test_keep.txt" if [ -f "$TEST_DIR/test_keep.txt" ] && [ -f "$TEST_DIR/test_keep.txt.zxc" ]; then log_pass "Keep source file (-k)" else log_fail "Keep source file failed (source was deleted)" fi # 19. Multi-Threading Tests (-T) echo "Testing Multi-Threading..." # 19.1 Single Thread (baseline) echo " Testing single thread (-T1)..." "$ZXC_BIN" -3 -T1 -c -k "$TEST_FILE_ARG" > "$TEST_DIR/test_T1.zxc" if [ ! -f "$TEST_DIR/test_T1.zxc" ]; then log_fail "Single thread compression failed" fi "$ZXC_BIN" -d -c "$TEST_DIR/test_T1.zxc" > "$TEST_DIR/test_T1.dec" if cmp -s "$TEST_FILE" "$TEST_DIR/test_T1.dec"; then SIZE_T1=$(wc -c < "$TEST_DIR/test_T1.zxc" | tr -d ' ') log_pass "Single thread (-T1, Size: $SIZE_T1)" else log_fail "Single thread round-trip failed" fi # 19.2 Multi-Thread (2 threads) echo " Testing 2 threads (-T2)..." "$ZXC_BIN" -3 -T2 -c -k "$TEST_FILE_ARG" > "$TEST_DIR/test_T2.zxc" if [ ! -f "$TEST_DIR/test_T2.zxc" ]; then log_fail "2-thread compression failed" fi "$ZXC_BIN" -d -c "$TEST_DIR/test_T2.zxc" > "$TEST_DIR/test_T2.dec" if cmp -s "$TEST_FILE" "$TEST_DIR/test_T2.dec"; then SIZE_T2=$(wc -c < "$TEST_DIR/test_T2.zxc" | tr -d ' ') log_pass "2 threads (-T2, Size: $SIZE_T2)" else log_fail "2-thread round-trip failed" fi # 19.3 Multi-Thread (all threads) echo " Testing all threads (-T0)..." "$ZXC_BIN" -3 -T0 -c -k "$TEST_FILE_ARG" > "$TEST_DIR/test_T0.zxc" if [ ! -f "$TEST_DIR/test_T0.zxc" ]; then log_fail "all-thread compression failed" fi "$ZXC_BIN" -d -c "$TEST_DIR/test_T0.zxc" > "$TEST_DIR/test_T0.dec" if cmp -s "$TEST_FILE" "$TEST_DIR/test_T0.dec"; then SIZE_T0=$(wc -c < "$TEST_DIR/test_T0.zxc" | tr -d ' ') log_pass "all threads (-T0, Size: $SIZE_T0)" else log_fail "all-thread round-trip failed" fi # 19.4 Verify determinism: All outputs should decompress to same content echo " Verifying thread-count independence..." if cmp -s "$TEST_DIR/test_T1.dec" "$TEST_DIR/test_T2.dec" && cmp -s "$TEST_DIR/test_T2.dec" "$TEST_DIR/test_T0.dec"; then log_pass "Deterministic output (all thread counts produce valid results)" else log_fail "Thread outputs differ (non-deterministic)" fi # 19.5 Cross-compatibility: File compressed with -T2 should decompress without -T flag echo " Testing cross-compatibility..." "$ZXC_BIN" -d -c "$TEST_DIR/test_T2.zxc" > "$TEST_DIR/test_T2_compat.dec" if cmp -s "$TEST_FILE" "$TEST_DIR/test_T2_compat.dec"; then log_pass "Cross-compatible decompression" else log_fail "Multi-threaded file not compatible with standard decompression" fi # 19.6 Large file with threads (performance validation) echo " Testing large file with threads..." dd if=/dev/zero bs=1M count=5 of="$TEST_DIR/test_large_mt.bin" 2>/dev/null "$ZXC_BIN" -3 -T4 -c -k "$TEST_DIR/test_large_mt.bin" > "$TEST_DIR/test_large_mt.zxc" "$ZXC_BIN" -d -c "$TEST_DIR/test_large_mt.zxc" > "$TEST_DIR/test_large_mt.dec" if cmp -s "$TEST_DIR/test_large_mt.bin" "$TEST_DIR/test_large_mt.dec"; then log_pass "Large file multi-threading" else log_fail "Large file multi-threading round-trip failed" fi # 20. JSON Output Tests echo "Testing JSON Output (-j, --json)..." # 20.1 List mode with JSON (single file) echo " Testing list mode with JSON (single file)..." "$ZXC_BIN" -z -k -f -C "$TEST_FILE_ARG" JSON_OUT=$("$ZXC_BIN" -l -j "$TEST_FILE_XC_ARG") if [[ "$JSON_OUT" == *'"filename"'* ]] && \ [[ "$JSON_OUT" == *'"compressed_size_bytes"'* ]] && \ [[ "$JSON_OUT" == *'"uncompressed_size_bytes"'* ]] && \ [[ "$JSON_OUT" == *'"compression_ratio"'* ]] && \ [[ "$JSON_OUT" == *'"format_version"'* ]] && \ [[ "$JSON_OUT" == *'"checksum_method"'* ]]; then log_pass "List mode JSON output (single file)" else log_fail "List mode JSON output missing expected fields" fi # 20.2 List mode with JSON (multiple files) echo " Testing list mode with JSON (multiple files)..." cp "$TEST_FILE_XC_ARG" "$TEST_DIR/test2.zxc" JSON_OUT=$("$ZXC_BIN" -l -j "$TEST_FILE_XC_ARG" "$TEST_DIR/test2.zxc") if [[ "$JSON_OUT" == "["* ]] && \ [[ "$JSON_OUT" == *"]" ]] && \ [[ "$JSON_OUT" == *'"filename"'* ]] && \ [[ "$JSON_OUT" == *","* ]]; then log_pass "List mode JSON output (multiple files - array)" else log_fail "List mode JSON output should produce array for multiple files" fi # 20.3 Benchmark mode with JSON echo " Testing benchmark mode with JSON..." JSON_OUT=$("$ZXC_BIN" -b1 -j "$TEST_FILE_ARG" 2>/dev/null) if [[ "$JSON_OUT" == *'"input_file"'* ]] && \ [[ "$JSON_OUT" == *'"input_size_bytes"'* ]] && \ [[ "$JSON_OUT" == *'"compressed_size_bytes"'* ]] && \ [[ "$JSON_OUT" == *'"compression_ratio"'* ]] && \ [[ "$JSON_OUT" == *'"duration_seconds"'* ]] && \ [[ "$JSON_OUT" == *'"compress_iterations"'* ]] && \ [[ "$JSON_OUT" == *'"decompress_iterations"'* ]] && \ [[ "$JSON_OUT" == *'"threads"'* ]] && \ [[ "$JSON_OUT" == *'"level"'* ]] && \ [[ "$JSON_OUT" == *'"checksum_enabled"'* ]] && \ [[ "$JSON_OUT" == *'"compress_speed_mbps"'* ]] && \ [[ "$JSON_OUT" == *'"decompress_speed_mbps"'* ]]; then log_pass "Benchmark mode JSON output" else log_fail "Benchmark mode JSON output missing expected fields" fi # 20.4 Integrity check with JSON (valid file) echo " Testing integrity check with JSON (valid file)..." "$ZXC_BIN" -z -k -f -C "$TEST_FILE_ARG" JSON_OUT=$("$ZXC_BIN" -t -j "$TEST_FILE_XC_ARG") if [[ "$JSON_OUT" == *'"filename"'* ]] && \ [[ "$JSON_OUT" == *'"status": "ok"'* ]] && \ [[ "$JSON_OUT" == *'"checksum_verified"'* ]]; then log_pass "Integrity check JSON output (valid file)" else log_fail "Integrity check JSON output missing expected fields" fi # 20.5 Integrity check with JSON (corrupt file) echo " Testing integrity check with JSON (corrupt file)..." "$ZXC_BIN" -z -k -f -C "$TEST_FILE_ARG" # Corrupt a byte printf '\xff' | dd of="$TEST_FILE_XC_ARG" bs=1 seek=100 count=1 conv=notrunc 2>/dev/null set +e JSON_OUT=$("$ZXC_BIN" -t -j "$TEST_FILE_XC_ARG" 2>&1) RET=$? set -e if [ $RET -ne 0 ] && \ [[ "$JSON_OUT" == *'"filename"'* ]] && \ [[ "$JSON_OUT" == *'"status": "failed"'* ]] && \ [[ "$JSON_OUT" == *'"error"'* ]]; then log_pass "Integrity check JSON output (corrupt file)" else log_fail "Integrity check JSON output for corrupt file incorrect" fi # 20.6 Verify JSON is parseable (if jq is available) echo " Checking JSON validity (if jq available)..." if command -v jq &> /dev/null; then "$ZXC_BIN" -z -k -f -C "$TEST_FILE_ARG" JSON_OUT=$("$ZXC_BIN" -l -j "$TEST_FILE_XC_ARG") if echo "$JSON_OUT" | jq . > /dev/null 2>&1; then log_pass "JSON output is valid (verified with jq)" else log_fail "JSON output is not valid JSON" fi else echo " [SKIP] jq not available, skipping JSON validation" fi # 21. Multiple Mode (-m) Tests echo "Testing Multiple Mode (-m)..." # 21.1 Compress multiple files echo " Testing compress multiple files..." cp "$TEST_FILE" "$TEST_DIR/multi1.txt" cp "$TEST_FILE" "$TEST_DIR/multi2.txt" "$ZXC_BIN" -m -3 "$TEST_DIR/multi1.txt" "$TEST_DIR/multi2.txt" if [ -f "$TEST_DIR/multi1.txt.zxc" ] && [ -f "$TEST_DIR/multi2.txt.zxc" ]; then log_pass "Compress multiple files (-m)" else log_fail "Compress multiple files failed" fi # 21.2 Decompress multiple files echo " Testing decompress multiple files..." rm -f "$TEST_DIR/multi1.txt" "$TEST_DIR/multi2.txt" "$ZXC_BIN" -d -m "$TEST_DIR/multi1.txt.zxc" "$TEST_DIR/multi2.txt.zxc" if cmp -s "$TEST_FILE" "$TEST_DIR/multi1.txt" && cmp -s "$TEST_FILE" "$TEST_DIR/multi2.txt"; then log_pass "Decompress multiple files (-d -m)" else log_fail "Decompress multiple files failed (content mismatch or missing files)" fi # 21.3 Error on multiple and stdout echo " Testing stdout restriction with multiple mode..." set +e "$ZXC_BIN" -m -c "$TEST_DIR/multi1.txt" "$TEST_DIR/multi2.txt" > /dev/null 2>&1 RET=$? set -e if [ $RET -ne 0 ]; then log_pass "Stdout rejected with multiple mode" else log_fail "Stdout should be rejected with multiple mode" fi # 22. Recursive Mode (-r) Tests echo "Testing Recursive Mode (-r)..." # Create a nested directory structure mkdir -p "$TEST_DIR/rec_test/subdir1" mkdir -p "$TEST_DIR/rec_test/subdir2" cp "$TEST_FILE" "$TEST_DIR/rec_test/fileA.txt" cp "$TEST_FILE" "$TEST_DIR/rec_test/subdir1/fileB.txt" cp "$TEST_FILE" "$TEST_DIR/rec_test/subdir2/fileC.txt" # 22.1 Compress recursively echo " Testing compress recursive directory..." "$ZXC_BIN" -r -3 "$TEST_DIR/rec_test" if [ -f "$TEST_DIR/rec_test/fileA.txt.zxc" ] && \ [ -f "$TEST_DIR/rec_test/subdir1/fileB.txt.zxc" ] && \ [ -f "$TEST_DIR/rec_test/subdir2/fileC.txt.zxc" ]; then log_pass "Compress recursive directory (-r)" else log_fail "Compress recursive directory failed" fi # 22.2 Decompress recursively echo " Testing decompress recursive directory..." rm -f "$TEST_DIR/rec_test/fileA.txt" "$TEST_DIR/rec_test/subdir1/fileB.txt" "$TEST_DIR/rec_test/subdir2/fileC.txt" "$ZXC_BIN" -d -r "$TEST_DIR/rec_test" if cmp -s "$TEST_FILE" "$TEST_DIR/rec_test/fileA.txt" && \ cmp -s "$TEST_FILE" "$TEST_DIR/rec_test/subdir1/fileB.txt" && \ cmp -s "$TEST_FILE" "$TEST_DIR/rec_test/subdir2/fileC.txt"; then log_pass "Decompress recursive directory (-d -r)" else log_fail "Decompress recursive directory failed (content mismatch or missing files)" fi # 23. Block Size Tests (-B) echo "Testing Block Size (-B)..." # 23.1 Round-trip with different block sizes for BS in 4K 64K 512K 1M; do echo " Testing block size -B $BS..." "$ZXC_BIN" -3 -B "$BS" -c -k "$TEST_FILE_ARG" > "$TEST_DIR/test_bs_${BS}.zxc" if [ ! -s "$TEST_DIR/test_bs_${BS}.zxc" ]; then log_fail "Block size $BS compression produced empty output" fi "$ZXC_BIN" -d -c "$TEST_DIR/test_bs_${BS}.zxc" > "$TEST_DIR/test_bs_${BS}.dec" if cmp -s "$TEST_FILE" "$TEST_DIR/test_bs_${BS}.dec"; then SIZE_BS=$(wc -c < "$TEST_DIR/test_bs_${BS}.zxc" | tr -d ' ') log_pass "Block size -B $BS (Size: $SIZE_BS)" else log_fail "Block size $BS round-trip failed" fi done # 23.2 Test suffix formats (KB, M, MB) echo " Testing block size suffix formats..." "$ZXC_BIN" -3 -B 128KB -c -k "$TEST_FILE_ARG" > "$TEST_DIR/test_bs_128KB.zxc" "$ZXC_BIN" -d -c "$TEST_DIR/test_bs_128KB.zxc" > "$TEST_DIR/test_bs_128KB.dec" if cmp -s "$TEST_FILE" "$TEST_DIR/test_bs_128KB.dec"; then log_pass "Block size suffix KB" else log_fail "Block size suffix KB round-trip failed" fi "$ZXC_BIN" -3 -B 2MB -c -k "$TEST_FILE_ARG" > "$TEST_DIR/test_bs_2MB.zxc" "$ZXC_BIN" -d -c "$TEST_DIR/test_bs_2MB.zxc" > "$TEST_DIR/test_bs_2MB.dec" if cmp -s "$TEST_FILE" "$TEST_DIR/test_bs_2MB.dec"; then log_pass "Block size suffix MB" else log_fail "Block size suffix MB round-trip failed" fi # 23.3 Error: invalid block size (not power of 2) echo " Testing invalid block size..." set +e "$ZXC_BIN" -3 -B 100K "$TEST_FILE_ARG" > /dev/null 2>&1 RET=$? set -e if [ $RET -ne 0 ]; then log_pass "Invalid block size rejected (100K)" else log_fail "Invalid block size should be rejected (100K is not power of 2)" fi # 23.4 Error: block size too small set +e "$ZXC_BIN" -3 -B 2K "$TEST_FILE_ARG" > /dev/null 2>&1 RET=$? set -e if [ $RET -ne 0 ]; then log_pass "Block size too small rejected (2K)" else log_fail "Block size 2K should be rejected (min is 4K)" fi # 23.5 Error: block size too large set +e "$ZXC_BIN" -3 -B 4M "$TEST_FILE_ARG" > /dev/null 2>&1 RET=$? set -e if [ $RET -ne 0 ]; then log_pass "Block size too large rejected (4M)" else log_fail "Block size 4M should be rejected (max is 2M)" fi # 24. Seekable Format (-S) echo "Testing Seekable Format (-S)..." # 24.1 Basic seekable round-trip echo " Testing basic seekable round-trip..." rm -f "$TEST_FILE_XC_ARG" "$TEST_FILE_DEC_BASH" "$ZXC_BIN" -S -c "$TEST_FILE_ARG" > "$TEST_FILE_XC_ARG" if [ ! -s "$TEST_FILE_XC_ARG" ]; then log_fail "Seekable compression failed" fi "$ZXC_BIN" -d -c "$TEST_FILE_XC_ARG" > "$TEST_FILE_DEC_BASH" if cmp -s "$TEST_FILE" "$TEST_FILE_DEC_BASH"; then log_pass "Seekable basic round-trip (-S)" else log_fail "Seekable decompression mismatch" fi # 24.2 Seekable file must be larger than normal (SEK block overhead) echo " Testing seekable overhead..." "$ZXC_BIN" -3 -c -k "$TEST_FILE_ARG" > "$TEST_DIR/normal.zxc" "$ZXC_BIN" -3 -S -c -k "$TEST_FILE_ARG" > "$TEST_DIR/seekable.zxc" SIZE_NORMAL=$(wc -c < "$TEST_DIR/normal.zxc" | tr -d ' ') SIZE_SEEKABLE=$(wc -c < "$TEST_DIR/seekable.zxc" | tr -d ' ') if [ "$SIZE_SEEKABLE" -gt "$SIZE_NORMAL" ]; then OVERHEAD=$((SIZE_SEEKABLE - SIZE_NORMAL)) log_pass "Seekable overhead verified (+${OVERHEAD} bytes)" else log_fail "Seekable file should be larger than normal (SEK block missing?)" fi # 24.3 Seekable with small block size (many blocks = larger seek table) echo " Testing seekable with small blocks (-B 4K)..." "$ZXC_BIN" -3 -S -B 4K -c -k "$TEST_FILE_ARG" > "$TEST_DIR/seekable_4k.zxc" "$ZXC_BIN" -d -c "$TEST_DIR/seekable_4k.zxc" > "$TEST_DIR/seekable_4k.dec" if cmp -s "$TEST_FILE" "$TEST_DIR/seekable_4k.dec"; then SIZE_4K=$(wc -c < "$TEST_DIR/seekable_4k.zxc" | tr -d ' ') # With 4K blocks, overhead should be larger than with default 256K blocks if [ "$SIZE_4K" -gt "$SIZE_SEEKABLE" ]; then log_pass "Seekable small blocks (-B 4K, Size: $SIZE_4K, more entries)" else log_pass "Seekable small blocks (-B 4K, Size: $SIZE_4K)" fi else log_fail "Seekable small blocks round-trip failed" fi # 24.4 Seekable with checksum echo " Testing seekable + checksum (-S -C)..." "$ZXC_BIN" -3 -S -C -c -k "$TEST_FILE_ARG" > "$TEST_DIR/seekable_chk.zxc" "$ZXC_BIN" -d -c "$TEST_DIR/seekable_chk.zxc" > "$TEST_DIR/seekable_chk.dec" if cmp -s "$TEST_FILE" "$TEST_DIR/seekable_chk.dec"; then log_pass "Seekable + checksum (-S -C)" else log_fail "Seekable + checksum round-trip failed" fi # 24.5 Seekable with multi-threading echo " Testing seekable + threads (-S -T2)..." "$ZXC_BIN" -3 -S -T2 -c -k "$TEST_FILE_ARG" > "$TEST_DIR/seekable_mt.zxc" "$ZXC_BIN" -d -c "$TEST_DIR/seekable_mt.zxc" > "$TEST_DIR/seekable_mt.dec" if cmp -s "$TEST_FILE" "$TEST_DIR/seekable_mt.dec"; then log_pass "Seekable + multi-threading (-S -T2)" else log_fail "Seekable + multi-threading round-trip failed" fi # 24.6 Seekable across all levels echo " Testing seekable across all levels..." SEEK_ALL_OK=1 for LEVEL in 1 2 3 4 5 6; do "$ZXC_BIN" -$LEVEL -S -c -k "$TEST_FILE_ARG" > "$TEST_DIR/seekable_lvl${LEVEL}.zxc" "$ZXC_BIN" -d -c "$TEST_DIR/seekable_lvl${LEVEL}.zxc" > "$TEST_DIR/seekable_lvl${LEVEL}.dec" if ! cmp -s "$TEST_FILE" "$TEST_DIR/seekable_lvl${LEVEL}.dec"; then SEEK_ALL_OK=0 log_fail "Seekable level $LEVEL round-trip failed" fi done if [ "$SEEK_ALL_OK" -eq 1 ]; then log_pass "Seekable across all levels (1-5)" fi # 24.7 Seekable pipe round-trip (no fseeko - validates SEK skip on stdin) echo " Testing seekable pipe round-trip..." cat "$TEST_FILE" | "$ZXC_BIN" -S -c | "$ZXC_BIN" -dc > "$TEST_DIR/seekable_pipe.dec" if cmp -s "$TEST_FILE" "$TEST_DIR/seekable_pipe.dec"; then log_pass "Seekable pipe round-trip" else log_fail "Seekable pipe round-trip content mismatch" fi # 24.8 Seekable + no-checksum echo " Testing seekable + no-checksum (-S -N)..." "$ZXC_BIN" -3 -S -N -c -k "$TEST_FILE_ARG" > "$TEST_DIR/seekable_nochk.zxc" "$ZXC_BIN" -d -c "$TEST_DIR/seekable_nochk.zxc" > "$TEST_DIR/seekable_nochk.dec" if cmp -s "$TEST_FILE" "$TEST_DIR/seekable_nochk.dec"; then log_pass "Seekable + no-checksum (-S -N)" else log_fail "Seekable + no-checksum round-trip failed" fi # 24.9 List command on seekable archive echo " Testing list command on seekable archive..." "$ZXC_BIN" -3 -S -C -k -f "$TEST_FILE_ARG" OUT=$("$ZXC_BIN" -l "$TEST_FILE_XC_ARG") if [[ "$OUT" == *"Compressed"* ]] && [[ "$OUT" == *"Uncompressed"* ]]; then log_pass "List command on seekable archive" else log_fail "List command on seekable archive failed" fi echo "All tests passed!" exit 0 zxc-0.11.0/tests/test_common.c000066400000000000000000000163311520102567100162350ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ #include "test_common.h" FILE* create_restricted_file(const char* path) { #ifdef _MSC_VER int fd = -1; _sopen_s(&fd, path, _O_CREAT | _O_WRONLY | _O_TRUNC, _SH_DENYNO, _S_IREAD | _S_IWRITE); return fd >= 0 ? _fdopen(fd, "w") : NULL; #else const int fd = open(path, O_CREAT | O_WRONLY | O_TRUNC, S_IRUSR | S_IWUSR); return fd >= 0 ? fdopen(fd, "w") : NULL; #endif } // Generates a buffer of random data (To force RAW) void gen_random_data(uint8_t* const buf, const size_t size) { for (size_t i = 0; i < size; i++) buf[i] = rand() & 0xFF; } // Generates repetitive data (To force GLO/GHI/LZ) void gen_lz_data(uint8_t* const buf, const size_t size) { const char* const pattern = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod " "tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim " "veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea " "commodo consequat. Duis aute irure dolor in reprehenderit in voluptate " "velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint " "occaecat cupidatat non proident, sunt in culpa qui officia deserunt " "mollit anim id est laborum."; const size_t pat_len = strlen(pattern); for (size_t i = 0; i < size; i++) buf[i] = pattern[i % pat_len]; } // Generates a regular numeric sequence (To force NUM) void gen_num_data(uint8_t* const buf, const size_t size) { // Fill with 32-bit integers uint32_t* const ptr = (uint32_t*)buf; const size_t count = size / 4; uint32_t val = 0; for (size_t i = 0; i < count; i++) { // Arithmetic sequence: 0, 100, 200... // Deltas are constant (100), perfect for NUM ptr[i] = val; val += 100; } } // Generates numeric sequence with 0 deltas (all identical) void gen_num_data_zero(uint8_t* const buf, const size_t size) { uint32_t* const ptr = (uint32_t*)buf; const size_t count = size / 4; for (size_t i = 0; i < count; i++) { ptr[i] = 42; } } // Generates numeric data with alternating small deltas (+1, -1) void gen_num_data_small(uint8_t* const buf, const size_t size) { uint32_t* const ptr = (uint32_t*)buf; const size_t count = size / 4; uint32_t val = 1000; for (size_t i = 0; i < count; i++) { ptr[i] = val; val += (i % 2 == 0) ? 1 : -1; } } // Generates numeric data with very large deltas to maximize bit width void gen_num_data_large(uint8_t* const buf, const size_t size) { uint32_t* const ptr = (uint32_t*)buf; const size_t count = size / 4; for (size_t i = 0; i < count; i++) { // Alternate between 0 and 0xFFFFFFFF (delta is huge) ptr[i] = (i % 2 == 0) ? 0 : 0xFFFFFFFF; } } void gen_binary_data(uint8_t* const buf, const size_t size) { // Pattern with problematic bytes that could be corrupted in text mode: // 0x0A (LF), 0x0D (CR), 0x00 (NULL), 0x1A (EOF/CTRL-Z), 0xFF const uint8_t pattern[] = { 0x5A, 0x58, 0x43, 0x00, // "ZXC" + NULL 0x0A, 0x0D, 0x0A, 0x00, // LF, CR, LF, NULL 0xFF, 0xFE, 0x0A, 0x0D, // High bytes + LF/CR 0x1A, 0x00, 0x0A, 0x0D, // EOF marker + NULL + LF/CR 0x00, 0x00, 0x0A, 0x0A, // Multiple NULLs and LFs }; const size_t pat_len = sizeof(pattern); for (size_t i = 0; i < size; i++) { buf[i] = pattern[i % pat_len]; } } // Generates data with small offsets (<=255 bytes) to force 1-byte offset encoding // This creates short repeating patterns with matches very close to each other void gen_small_offset_data(uint8_t* const buf, const size_t size) { // Create short repeating patterns with very short distances. // Uses a 5-byte period (not aligned to uint32_t) to avoid being // classified as NUM data by zxc_probe_is_numeric(). // LZ will match at offset=5 (< 255), exercising 8-bit offset encoding. const uint8_t pattern[] = "ABCDE"; for (size_t i = 0; i < size; i++) { buf[i] = pattern[i % 5]; } } // Generates data with large offsets (>255 bytes) to force 2-byte offset encoding // This creates patterns where matches are far apart void gen_large_offset_data(uint8_t* const buf, const size_t size) { // First 300 bytes: unique random data (no matches possible) for (size_t i = 0; i < 300 && i < size; i++) { buf[i] = (uint8_t)((i * 7 + 13) % 256); } // Then: repeat patterns from the beginning (offset > 255) for (size_t i = 300; i < size; i++) { buf[i] = buf[i - 300]; // Offset of 300 bytes (requires 2-byte encoding) } } void fill_seek_data(uint8_t* buf, size_t size, uint8_t seed) { for (size_t i = 0; i < size; i++) { buf[i] = (uint8_t)(seed + (i * 17) + (i >> 8)); } } // Generic Round-Trip test function (Compress -> Decompress -> Compare) int test_round_trip(const char* test_name, const uint8_t* input, size_t size, int level, int checksum_enabled) { printf("=== TEST: %s (Sz: %zu, Lvl: %d, CRC: %s) ===\n", test_name, size, level, checksum_enabled ? "Enabled" : "Disabled"); FILE* const f_in = tmpfile(); FILE* const f_comp = tmpfile(); FILE* const f_decomp = tmpfile(); if (!f_in || !f_comp || !f_decomp) { perror("tmpfile"); if (f_in) fclose(f_in); if (f_comp) fclose(f_comp); if (f_decomp) fclose(f_decomp); return 0; } fwrite(input, 1, size, f_in); fseek(f_in, 0, SEEK_SET); zxc_compress_opts_t _sco1 = { .n_threads = 1, .level = level, .checksum_enabled = checksum_enabled}; if (zxc_stream_compress(f_in, f_comp, &_sco1) < 0) { printf("Compression Failed!\n"); fclose(f_in); fclose(f_comp); fclose(f_decomp); return 0; } long comp_size = ftell(f_comp); printf("Compressed Size: %ld (Ratio: %.2f)\n", comp_size, (double)size / (comp_size > 0 ? comp_size : 1)); fseek(f_comp, 0, SEEK_SET); zxc_decompress_opts_t _sdo2 = {.n_threads = 1, .checksum_enabled = checksum_enabled}; if (zxc_stream_decompress(f_comp, f_decomp, &_sdo2) < 0) { printf("Decompression Failed!\n"); fclose(f_in); fclose(f_comp); fclose(f_decomp); return 0; } long decomp_size = ftell(f_decomp); if (decomp_size != (long)size) { printf("Size Mismatch! Expected %zu, got %ld\n", size, decomp_size); fclose(f_in); fclose(f_comp); fclose(f_decomp); return 0; } fseek(f_decomp, 0, SEEK_SET); uint8_t* out_buf = malloc(size > 0 ? size : 1); if (fread(out_buf, 1, size, f_decomp) != size) { printf("Read validation failed (incomplete read)!\n"); free(out_buf); fclose(f_in); fclose(f_comp); fclose(f_decomp); return 0; } if (size > 0 && memcmp(input, out_buf, size) != 0) { printf("Data Mismatch (Content Corruption)!\n"); free(out_buf); fclose(f_in); fclose(f_comp); fclose(f_decomp); return 0; } printf("PASS\n\n"); free(out_buf); fclose(f_in); fclose(f_comp); fclose(f_decomp); return 1; } zxc-0.11.0/tests/test_common.h000066400000000000000000000133551520102567100162450ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ #ifndef ZXC_TEST_COMMON_H #define ZXC_TEST_COMMON_H #include #include #include #include #include #include #ifdef _MSC_VER #include #include #endif #include "../include/zxc_buffer.h" #include "../include/zxc_error.h" #include "../include/zxc_pstream.h" #include "../include/zxc_sans_io.h" #include "../include/zxc_seekable.h" #include "../include/zxc_stream.h" #include "../src/lib/zxc_internal.h" /* Declarative test-table entry: TEST_CASE(test_foo) -> { "test_foo", test_foo } */ #define TEST_CASE(fn) { #fn, fn } /* --- IO helper ---------------------------------------------------------- */ /* Creates a temporary file with restricted permissions (0600). * Returns a FILE* opened for writing, or NULL on failure. */ FILE* create_restricted_file(const char* path); /* --- Data generators ---------------------------------------------------- */ void gen_random_data(uint8_t* buf, size_t size); void gen_lz_data(uint8_t* buf, size_t size); void gen_num_data(uint8_t* buf, size_t size); void gen_num_data_zero(uint8_t* buf, size_t size); void gen_num_data_small(uint8_t* buf, size_t size); void gen_num_data_large(uint8_t* buf, size_t size); void gen_binary_data(uint8_t* buf, size_t size); void gen_small_offset_data(uint8_t* buf, size_t size); void gen_large_offset_data(uint8_t* buf, size_t size); void fill_seek_data(uint8_t* buf, size_t size, uint8_t seed); /* Generic streaming round-trip check (compress, decompress, compare). * Returns 1 on success, 0 on failure. */ int test_round_trip(const char* test_name, const uint8_t* input, size_t size, int level, int checksum_enabled); /* --- Test function prototypes ------------------------------------------ */ /* Buffer API */ int test_buffer_api(void); int test_buffer_api_scratch_buf(void); int test_buffer_error_codes(void); int test_get_decompressed_size(void); int test_decompress_fast_vs_safe_path(void); int test_max_compressed_size_logic(void); /* Block API */ int test_block_api(void); int test_block_api_boundary_sizes(void); int test_block_api_large_block_varint(void); int test_decompress_block_safe(void); int test_decompress_block_bound(void); /* Context API */ int test_opaque_context_api(void); int test_estimate_cctx_size(void); /* Stream API */ int test_null_output_decompression(void); int test_invalid_arguments(void); int test_truncated_input(void); int test_io_failures(void); int test_thread_params(void); int test_multithread_roundtrip(void); int test_stream_get_decompressed_size_errors(void); int test_stream_engine_errors(void); /* Push Streaming API (zxc_pstream.h) */ int test_pstream_roundtrip_basic(void); int test_pstream_roundtrip_no_checksum(void); int test_pstream_roundtrip_levels(void); int test_pstream_tiny_chunks(void); int test_pstream_drip_one_byte(void); int test_pstream_empty_input(void); int test_pstream_large_random(void); int test_pstream_compatible_with_buffer_api(void); int test_pstream_decompress_compatible_with_buffer_api(void); int test_pstream_invalid_args(void); int test_pstream_truncated_input(void); int test_pstream_corrupted_magic(void); int test_pstream_decode_seekable_archive(void); int test_pstream_compress_after_end_rejected(void); int test_pstream_compress_drain_block_resume(void); /* Stream round-trip coverage (patterns x sizes x levels x checksum) */ int test_roundtrip_raw_random(void); int test_roundtrip_ghi_text(void); int test_roundtrip_glo_text(void); int test_roundtrip_num_seq(void); int test_roundtrip_num_zero(void); int test_roundtrip_num_small(void); int test_roundtrip_num_large(void); int test_roundtrip_small_50(void); int test_roundtrip_empty(void); int test_roundtrip_1byte(void); int test_roundtrip_1byte_crc(void); int test_roundtrip_large_15mb_lz(void); int test_roundtrip_large_15mb_num(void); int test_roundtrip_checksum_off(void); int test_roundtrip_checksum_on(void); int test_roundtrip_level1(void); int test_roundtrip_level2(void); int test_roundtrip_level3(void); int test_roundtrip_level4(void); int test_roundtrip_level5(void); int test_roundtrip_level6(void); int test_roundtrip_binary(void); int test_roundtrip_binary_crc(void); int test_roundtrip_binary_small(void); int test_roundtrip_offset8_small(void); int test_roundtrip_offset8_lvl5(void); int test_roundtrip_offset16_large(void); int test_roundtrip_offset16_lvl5(void); int test_roundtrip_offset_mixed(void); /* Seekable (single-threaded) */ int test_seekable_table_sizes(void); int test_seekable_table_write(void); int test_seekable_roundtrip(void); int test_seekable_open_query(void); int test_seekable_random_access(void); int test_seekable_non_seekable_reject(void); int test_seekable_single_block(void); int test_seekable_all_levels(void); int test_seekable_many_blocks(void); int test_seekable_open_file(void); int test_seekable_cross_boundary(void); int test_seekable_truncated_input(void); int test_seekable_corrupted_sek(void); int test_seekable_range_out_of_bounds(void); int test_seekable_dst_too_small(void); int test_seekable_empty_file(void); int test_seekable_no_checksum(void); int test_seekable_with_checksum(void); /* Seekable (multi-threaded) */ int test_seekable_mt_roundtrip(void); int test_seekable_mt_single_block(void); int test_seekable_mt_random_access(void); int test_seekable_mt_full_file(void); /* Format (on-disk) */ int test_bit_reader(void); int test_bitpack(void); int test_huffman_codec(void); int test_eof_block_structure(void); int test_header_checksum(void); int test_global_checksum_order(void); int test_legacy_header(void); /* Misc */ int test_error_name(void); int test_library_info_api(void); #endif /* ZXC_TEST_COMMON_H */ zxc-0.11.0/tests/test_context_api.c000066400000000000000000000141451520102567100172630ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ #include "test_common.h" int test_opaque_context_api() { printf("=== TEST: Opaque Context API (zxc_create_cctx / zxc_create_dctx) ===\n"); /* 1. NULL context -> ZXC_ERROR_NULL_INPUT */ { uint8_t d[64]; zxc_compress_opts_t co = {.level = 3, .checksum_enabled = 0}; if (zxc_compress_cctx(NULL, d, sizeof(d), d, sizeof(d), &co) != ZXC_ERROR_NULL_INPUT) { printf(" [FAIL] compress_cctx NULL ctx\n"); return 0; } zxc_decompress_opts_t do_ = {.checksum_enabled = 0}; if (zxc_decompress_dctx(NULL, d, sizeof(d), d, sizeof(d), &do_) != ZXC_ERROR_NULL_INPUT) { printf(" [FAIL] decompress_dctx NULL ctx\n"); return 0; } printf(" [PASS] NULL context -> ZXC_ERROR_NULL_INPUT\n"); } /* 2. Create with eager init, multi-call reuse, free */ zxc_compress_opts_t create_opts = {.level = 3, .checksum_enabled = 0}; zxc_cctx* cctx = zxc_create_cctx(&create_opts); zxc_dctx* dctx = zxc_create_dctx(); if (!cctx || !dctx) { printf(" [FAIL] create returned NULL\n"); zxc_free_cctx(cctx); zxc_free_dctx(dctx); return 0; } const size_t src_sz = 8192; uint8_t* src = malloc(src_sz); const size_t comp_cap = (size_t)zxc_compress_bound(src_sz); uint8_t* comp = malloc(comp_cap); uint8_t* dec = malloc(src_sz); /* 3. Three calls with the SAME cctx: level 1, 3, 5 */ for (int lvl = 1; lvl <= 5; lvl += 2) { gen_lz_data(src, src_sz); zxc_compress_opts_t co = {.level = lvl, .checksum_enabled = (lvl == 3)}; const int64_t csz = zxc_compress_cctx(cctx, src, src_sz, comp, comp_cap, &co); if (csz <= 0) { printf(" [FAIL] compress_cctx level %d returned %lld\n", lvl, (long long)csz); goto fail; } zxc_decompress_opts_t do_ = {.checksum_enabled = (lvl == 3)}; const int64_t dsz = zxc_decompress_dctx(dctx, comp, (size_t)csz, dec, src_sz, &do_); if (dsz != (int64_t)src_sz || memcmp(src, dec, src_sz) != 0) { printf(" [FAIL] roundtrip level %d (dsz=%lld)\n", lvl, (long long)dsz); goto fail; } } printf(" [PASS] Multi-call reuse (level 1, 3, 5)\n"); /* 4. Free is safe to call multiple times / on NULL */ zxc_free_cctx(cctx); cctx = NULL; zxc_free_cctx(NULL); /* no-op */ zxc_free_dctx(dctx); dctx = NULL; zxc_free_dctx(NULL); printf(" [PASS] Free + double-free + NULL safety\n"); free(src); free(comp); free(dec); printf("PASS\n\n"); return 1; fail: zxc_free_cctx(cctx); zxc_free_dctx(dctx); free(src); free(comp); free(dec); return 0; } int test_estimate_cctx_size() { printf("=== TEST: Unit - zxc_estimate_cctx_size ===\n"); const int LVL = 3; /* 1. Zero input returns zero. */ if (zxc_estimate_cctx_size(0, LVL) != 0) { printf(" [FAIL] estimate(0) must return 0\n"); return 0; } printf(" [PASS] estimate(0) == 0\n"); /* 2. Non-zero input returns non-zero estimate. */ const uint64_t e1k = zxc_estimate_cctx_size(1024, LVL); if (e1k == 0) { printf(" [FAIL] estimate(1 KiB) must be > 0\n"); return 0; } printf(" [PASS] estimate(1 KiB) = %llu bytes\n", (unsigned long long)e1k); /* 3. Sizes below ZXC_BLOCK_SIZE_MIN collapse to the same estimate. */ if (zxc_estimate_cctx_size(512, LVL) != e1k || zxc_estimate_cctx_size(4096, LVL) != e1k) { printf(" [FAIL] estimates below MIN must round to ZXC_BLOCK_SIZE_MIN\n"); return 0; } printf(" [PASS] estimate rounds sub-MIN inputs to the same value\n"); /* 4. Monotonic: estimate grows with src_size across block_size tiers. */ const uint64_t e64k = zxc_estimate_cctx_size(64 * 1024, LVL); const uint64_t e1m = zxc_estimate_cctx_size(1024 * 1024, LVL); const uint64_t e8m = zxc_estimate_cctx_size(8 * 1024 * 1024, LVL); if (!(e1k <= e64k && e64k <= e1m && e1m <= e8m)) { printf(" [FAIL] estimates must be monotonic: %llu, %llu, %llu, %llu\n", (unsigned long long)e1k, (unsigned long long)e64k, (unsigned long long)e1m, (unsigned long long)e8m); return 0; } printf(" [PASS] monotonic: 1K=%llu, 64K=%llu, 1M=%llu, 8M=%llu\n", (unsigned long long)e1k, (unsigned long long)e64k, (unsigned long long)e1m, (unsigned long long)e8m); /* 5. Sanity: estimate for a large block must exceed the block itself. */ if (e1m < 1024 * 1024) { printf(" [FAIL] estimate(1 MiB)=%llu should exceed 1 MiB\n", (unsigned long long)e1m); return 0; } printf(" [PASS] estimate exceeds raw block size (context overhead present)\n"); /* 6. Sub-linear scaling: doubling src_size must roughly double the estimate, * bounded above by 10x factor (chain 2x, everything else 1x). */ if (e8m < 4 * e1m || e8m > 12 * e1m) { printf(" [FAIL] 8x src_size yields %.2fx memory (expected ~8x)\n", (double)e8m / (double)e1m); return 0; } printf(" [PASS] scaling: 8x src_size -> %.2fx memory\n", (double)e8m / (double)e1m); /* 7. Level 6 includes the optimal-parser scratch peak (~18 x chunk_size) * on top of the persistent cctx, so it must exceed the level-3 figure * by at least one chunk_size worth of bytes. */ const uint64_t e1m_l3 = zxc_estimate_cctx_size(1024 * 1024, 3); const uint64_t e1m_l6 = zxc_estimate_cctx_size(1024 * 1024, 6); if (e1m_l6 <= e1m_l3 + (1024 * 1024)) { printf(" [FAIL] level 6 must add optimal-parser scratch: l3=%llu l6=%llu\n", (unsigned long long)e1m_l3, (unsigned long long)e1m_l6); return 0; } printf(" [PASS] level 6 vs level 3 at 1 MiB: l3=%llu l6=%llu (delta=%llu)\n", (unsigned long long)e1m_l3, (unsigned long long)e1m_l6, (unsigned long long)(e1m_l6 - e1m_l3)); printf("PASS\n\n"); return 1; } zxc-0.11.0/tests/test_format.c000066400000000000000000000372061520102567100162410ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ #include "test_common.h" /* * Test for zxc_br_init and zxc_br_ensure */ int test_bit_reader() { printf("=== TEST: Unit - Bit Reader (zxc_br_init / zxc_br_ensure) ===\n"); // Case 1: Normal initialization uint8_t buffer[16]; for (int i = 0; i < 16; i++) buffer[i] = (uint8_t)i; zxc_bit_reader_t br; zxc_br_init(&br, buffer, 16); if (br.bits != 64) return 0; if (br.ptr != buffer + 8) return 0; if (br.accum != zxc_le64(buffer)) return 0; printf(" [PASS] Normal init\n"); // Case 2: Small buffer initialization (should not crash) uint8_t small_buffer[4] = {0xAA, 0xBB, 0xCC, 0xDD}; zxc_br_init(&br, small_buffer, 4); // Should have read 4 bytes safely (in LE order, matching zxc_le_partial) uint64_t expected_accum = (uint64_t)small_buffer[0] | ((uint64_t)small_buffer[1] << 8) | ((uint64_t)small_buffer[2] << 16) | ((uint64_t)small_buffer[3] << 24); if (br.accum != expected_accum) return 0; if (br.ptr != small_buffer + 4) return 0; printf(" [PASS] Small buffer init\n"); // Case 3: zxc_br_ensure (Normal refill) zxc_br_init(&br, buffer, 16); br.bits = 10; // Simulate consumption br.accum >>= 54; // Simulate shift zxc_br_ensure(&br, 32); // Should have refilled if (br.bits < 32) return 0; printf(" [PASS] Ensure normal refill\n"); // Case 4: zxc_br_ensure (End of stream) // Init with full buffer but advanced pointer near end zxc_br_init(&br, buffer, 16); br.ptr = buffer + 16; // At end br.bits = 0; // Try to ensure bits, should not read past end zxc_br_ensure(&br, 10); // The key is it didn't crash. printf(" [PASS] Ensure EOF safety\n"); printf("PASS\n\n"); return 1; } /* * Test for zxc_bitpack_stream_32 */ int test_bitpack() { printf("=== TEST: Unit - Bit Packing (zxc_bitpack_stream_32) ===\n"); const uint32_t src[4] = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}; uint8_t dst[16]; // Pack 4 values with 4 bits each. // Input is 0xFFFFFFFF, but should be masked to 0xF (1111). // Result should be 2 bytes: 0xFF, 0xFF int len = zxc_bitpack_stream_32(src, 4, dst, 16, 4); if (len != 2) return 0; if (dst[0] != 0xFF || dst[1] != 0xFF) return 0; printf(" [PASS] Bitpack overflow masking\n"); // Edge case: bits = 32 const uint32_t src32[1] = {0x12345678}; len = zxc_bitpack_stream_32(src32, 1, dst, 16, 32); if (len != 4) return 0; if (zxc_le32(dst) != 0x12345678) return 0; printf(" [PASS] Bitpack 32 bits\n"); printf("PASS\n\n"); return 1; } /* Round-trip the Huffman codec over a few representative literal distributions. */ static int huf_roundtrip_case(const char* label, const uint8_t* literals, size_t n) { uint32_t freq[ZXC_HUF_NUM_SYMBOLS] = {0}; for (size_t i = 0; i < n; i++) freq[literals[i]]++; uint8_t code_len[ZXC_HUF_NUM_SYMBOLS]; if (zxc_huf_build_code_lengths(freq, code_len, NULL) != ZXC_OK) { printf("Failed [%s]: build_code_lengths\n", label); return 0; } /* Validate the lengths-limit invariant. */ for (int i = 0; i < ZXC_HUF_NUM_SYMBOLS; i++) { if (code_len[i] > ZXC_HUF_MAX_CODE_LEN) { printf("Failed [%s]: code_len[%d] = %d > %d\n", label, i, code_len[i], ZXC_HUF_MAX_CODE_LEN); return 0; } } /* Worst-case payload size: 134-byte header + n bytes (RAW upper bound). */ const size_t cap = ZXC_HUF_HEADER_SIZE + n + 64; uint8_t* enc = (uint8_t*)malloc(cap); uint8_t* dec = (uint8_t*)malloc(n); if (!enc || !dec) { free(enc); free(dec); printf("Failed [%s]: alloc\n", label); return 0; } const int written = zxc_huf_encode_section(literals, n, code_len, enc, cap); if (written < 0) { free(enc); free(dec); printf("Failed [%s]: encode_section -> %d\n", label, written); return 0; } const int rc = zxc_huf_decode_section(enc, (size_t)written, dec, n); if (rc != ZXC_OK) { free(enc); free(dec); printf("Failed [%s]: decode_section -> %d\n", label, rc); return 0; } if (memcmp(literals, dec, n) != 0) { free(enc); free(dec); printf("Failed [%s]: roundtrip mismatch\n", label); return 0; } free(enc); free(dec); printf(" [PASS] %s (n=%zu, encoded=%d B, ratio=%.1f%%)\n", label, n, written, 100.0 * (double)written / (double)n); return 1; } int test_huffman_codec() { printf("=== TEST: Unit - Huffman Codec (build/encode/decode roundtrip) ===\n"); const size_t N = 8192; uint8_t* buf = (uint8_t*)malloc(N); if (!buf) return 0; /* Case 1: heavily skewed (90% one byte, 10% noise). */ for (size_t i = 0; i < N; i++) buf[i] = (rand() % 10 == 0) ? (uint8_t)(rand() & 0xFF) : 'A'; if (!huf_roundtrip_case("Skewed (90% 'A')", buf, N)) { free(buf); return 0; } /* Case 2: uniform random - Huffman should be near no-op (~1 byte/sym). */ for (size_t i = 0; i < N; i++) buf[i] = (uint8_t)(rand() & 0xFF); if (!huf_roundtrip_case("Uniform random", buf, N)) { free(buf); return 0; } /* Case 3: two-symbol alphabet - best case, ~1 bit/symbol. */ for (size_t i = 0; i < N; i++) buf[i] = (rand() & 1) ? 'X' : 'Y'; if (!huf_roundtrip_case("Two-symbol alphabet", buf, N)) { free(buf); return 0; } /* Case 4: single-symbol - degenerate but must still roundtrip. */ for (size_t i = 0; i < N; i++) buf[i] = 'Z'; if (!huf_roundtrip_case("Single-symbol", buf, N)) { free(buf); return 0; } /* Case 5: small block (just above the min-literals threshold). */ for (size_t i = 0; i < ZXC_HUF_MIN_LITERALS; i++) buf[i] = (rand() % 4 == 0) ? (uint8_t)(rand() & 0xFF) : 'k'; if (!huf_roundtrip_case("Small block at threshold", buf, ZXC_HUF_MIN_LITERALS)) { free(buf); return 0; } free(buf); printf("PASS\n\n"); return 1; } // Checks that the EOF block is correctly appended int test_eof_block_structure() { printf("=== TEST: Unit - EOF Block Structure ===\n"); const char* input = "test"; size_t src_size = 4; size_t max_dst_size = (size_t)zxc_compress_bound(src_size); uint8_t* compressed = malloc(max_dst_size); if (!compressed) return 0; zxc_compress_opts_t _co26 = {.level = 1, .checksum_enabled = 0}; int64_t comp_size = zxc_compress(input, src_size, compressed, max_dst_size, &_co26); if (comp_size <= 0) { printf("Failed: Compression returned 0\n"); free(compressed); return 0; } // Validating Footer and EOF Block // Total Overhead: 12 bytes (Footer) + 8 bytes (EOF Header) = 20 bytes if (comp_size < 20) { printf("Failed: Compressed size too small for Footer + EOF (%lld)\n", (long long)comp_size); free(compressed); return 0; } // 1. Verify 12-byte Footer // Structure: [SrcSize (8)] + [Hash (4)] const uint8_t* footer_ptr = compressed + comp_size - 12; uint32_t f_src_low = zxc_le32(footer_ptr); // Should be 4 uint32_t f_src_high = zxc_le32(footer_ptr + 4); // Should be 0 uint32_t f_hash = zxc_le32(footer_ptr + 8); // Should be 0 (checksum disabled) if (f_src_low != 4 || f_src_high != 0 || f_hash != 0) { printf("Failed: Footer mismatch. Src: %u, Hash: %u\n", f_src_low, f_hash); free(compressed); return 0; } // 2. Verify EOF Block Header (8 bytes) // Should be immediately before the footer const uint8_t* eof_ptr = compressed + comp_size - 20; uint8_t expected[8] = {0xFF, 0, 0, 0, 0, 0, 0, 0}; expected[7] = zxc_hash8(expected); if (memcmp(eof_ptr, expected, 8) != 0) { printf( "Failed: EOF block mismatch.\nExpected: %02X %02X %02X ... %02X\nGot: %02X %02X " "%02X ... %02X\n", expected[0], expected[1], expected[2], expected[7], eof_ptr[0], eof_ptr[1], eof_ptr[2], eof_ptr[7]); free(compressed); return 0; } printf("PASS\n\n"); free(compressed); return 1; } int test_header_checksum() { printf("Running test_header_checksum...\n"); uint8_t header_buf[ZXC_BLOCK_HEADER_SIZE]; zxc_block_header_t bh_in = {.block_type = ZXC_BLOCK_GLO, .block_flags = 0, .reserved = 0, .header_crc = 0, .comp_size = 1024}; // 1. Write Header if (zxc_write_block_header(header_buf, ZXC_BLOCK_HEADER_SIZE, &bh_in) != ZXC_BLOCK_HEADER_SIZE) { printf(" [FAIL] zxc_write_block_header failed\n"); return 0; } // Verify manually that checksum byte is non-zero (highly likely) if (header_buf[7] == 0) { // It's technically possible but very unlikely with a good hash printf(" [WARN] Checksum is 0 (unlikely but possible)\n"); } // 2. Read Header (Valid) zxc_block_header_t bh_out; if (zxc_read_block_header(header_buf, ZXC_BLOCK_HEADER_SIZE, &bh_out) != 0) { printf(" [FAIL] zxc_read_block_header failed on valid input\n"); return 0; } if (bh_out.block_type != bh_in.block_type || bh_out.comp_size != bh_in.comp_size || bh_out.header_crc != header_buf[7]) { printf(" [FAIL] Read data mismatch\n"); return 0; } // 3. Corrupt Header Checksum uint8_t original_crc = header_buf[7]; header_buf[7] = ~original_crc; // Flip bits if (zxc_read_block_header(header_buf, ZXC_BLOCK_HEADER_SIZE, &bh_out) == 0) { printf(" [FAIL] zxc_read_block_header should have failed on corrupted CRC\n"); return 0; } header_buf[7] = original_crc; // Restore // 4. Corrupt Header Content header_buf[0] = ZXC_BLOCK_RAW; // Change type if (zxc_read_block_header(header_buf, ZXC_BLOCK_HEADER_SIZE, &bh_out) == 0) { printf(" [FAIL] zxc_read_block_header should have failed on corrupted content\n"); return 0; } printf("PASS\n\n"); return 1; } // 5. Test Global Checksum Order Sensitivity // Ensures that swapping two blocks (even if valid individually) triggers a global checksum failure. int test_global_checksum_order() { printf("TEST: Global Checksum Order Sensitivity... "); // 1. Create input data withDISTINCT patterns for 2 blocks (so blocks are different) // ZXC_BLOCK_SIZE_DEFAULT is 256KB. We need > 256KB. Let's use 600KB. size_t input_sz = 600 * 1024; uint8_t* val_buf = malloc(input_sz); if (!val_buf) return 0; // Fill Block 1 with 0xAA, Block 2 with 0xBB, Block 3 with 0xCC... memset(val_buf, 0xAA, 256 * 1024); memset(val_buf + 256 * 1024, 0xBB, 256 * 1024); memset(val_buf + 512 * 1024, 0xCC, input_sz - 512 * 1024); FILE* f_in = tmpfile(); FILE* f_comp = tmpfile(); fwrite(val_buf, 1, input_sz, f_in); rewind(f_in); // 2. Compress with Checksum Enabled zxc_compress_opts_t _sco27 = {.n_threads = 1, .level = 1, .checksum_enabled = 1}; zxc_stream_compress(f_in, f_comp, &_sco27); // 3. Read compressed data to memory long comp_sz = ftell(f_comp); rewind(f_comp); uint8_t* comp_buf = malloc((size_t)comp_sz); if (fread(comp_buf, 1, comp_sz, f_comp) != (size_t)comp_sz) { printf("[FAIL] Failed to read compressed data\n"); free(val_buf); free(comp_buf); fclose(f_in); fclose(f_comp); return 0; } // 4. Parse Blocks to identify Block 1 and Block 2 // File Header: ZXC_FILE_HEADER_SIZE bytes size_t off1 = ZXC_FILE_HEADER_SIZE; // Parse Block 1 Header zxc_block_header_t bh1; zxc_read_block_header(comp_buf + off1, ZXC_BLOCK_HEADER_SIZE, &bh1); size_t len1 = ZXC_BLOCK_HEADER_SIZE + bh1.comp_size + ZXC_BLOCK_CHECKSUM_SIZE; size_t off2 = off1 + len1; // Parse Block 2 Header zxc_block_header_t bh2; zxc_read_block_header(comp_buf + off2, ZXC_BLOCK_HEADER_SIZE, &bh2); size_t len2 = ZXC_BLOCK_HEADER_SIZE + bh2.comp_size + ZXC_BLOCK_CHECKSUM_SIZE; // Ensure we have at least 2 full blocks + EOF + Global Checksum if (off2 + len2 > (size_t)comp_sz) { printf("[FAIL] Compressed size too small for test\n"); free(val_buf); free(comp_buf); fclose(f_in); fclose(f_comp); return 0; } // 5. Swap Block 1 and Block 2 // To safely swap, we need a new buffer uint8_t* swapped_buf = malloc((size_t)comp_sz); // Copy File Header // Copy File Header memcpy(swapped_buf, comp_buf, ZXC_FILE_HEADER_SIZE); size_t w_off = ZXC_FILE_HEADER_SIZE; // Write Block 2 first memcpy(swapped_buf + w_off, comp_buf + off2, len2); w_off += len2; // Write Block 1 second memcpy(swapped_buf + w_off, comp_buf + off1, len1); w_off += len1; // Write remaining data (EOF block + Global Checksum) size_t remaining_off = off2 + len2; size_t remaining_len = comp_sz - remaining_off; memcpy(swapped_buf + w_off, comp_buf + remaining_off, remaining_len); // 6. Write to File for Decompression FILE* f_bad = tmpfile(); fwrite(swapped_buf, 1, (size_t)comp_sz, f_bad); rewind(f_bad); // 7. Attempt Decompression FILE* f_out = tmpfile(); zxc_decompress_opts_t _sdo28 = {.n_threads = 1, .checksum_enabled = 1}; int64_t res = zxc_stream_decompress(f_bad, f_out, &_sdo28); fclose(f_in); fclose(f_comp); fclose(f_bad); fclose(f_out); free(val_buf); free(comp_buf); free(swapped_buf); if (res >= 0) { printf(" [FAIL] zxc_stream_decompress unexpectedly succeeded on swapped blocks\n"); return 0; } printf("PASS\n\n"); return 1; } int test_legacy_header() { printf("=== TEST: Legacy header (chunk_size_code=64) ===\n"); // Build a valid file header with legacy chunk_size_code = 64 (= 256 KB) uint8_t hdr[ZXC_FILE_HEADER_SIZE]; memset(hdr, 0, sizeof(hdr)); // Magic word (LE) hdr[0] = 0xF5; hdr[1] = 0x2E; hdr[2] = 0xB0; hdr[3] = 0x9C; // Version hdr[4] = ZXC_FILE_FORMAT_VERSION; // Legacy chunk size code hdr[5] = 64; // Flags: no checksum hdr[6] = 0; // Compute CRC16 (bytes 14-15 zeroed, then hash) hdr[14] = 0; hdr[15] = 0; uint16_t crc = zxc_hash16(hdr); hdr[14] = (uint8_t)(crc & 0xFF); hdr[15] = (uint8_t)(crc >> 8); size_t block_size = 0; int has_checksum = -1; int rc = zxc_read_file_header(hdr, sizeof(hdr), &block_size, &has_checksum); if (rc != ZXC_OK) { printf(" [FAIL] zxc_read_file_header returned %d (%s)\n", rc, zxc_error_name(rc)); return 0; } if (block_size != 256 * 1024) { printf(" [FAIL] block_size = %zu, expected %d\n", block_size, 256 * 1024); return 0; } if (has_checksum != 0) { printf(" [FAIL] has_checksum = %d, expected 0\n", has_checksum); return 0; } printf(" [PASS] Legacy code 64 -> block_size = 256 KB\n"); // Verify that invalid codes are rejected hdr[5] = 99; // Not a valid exponent nor legacy value hdr[14] = 0; hdr[15] = 0; crc = zxc_hash16(hdr); hdr[14] = (uint8_t)(crc & 0xFF); hdr[15] = (uint8_t)(crc >> 8); rc = zxc_read_file_header(hdr, sizeof(hdr), &block_size, &has_checksum); if (rc != ZXC_ERROR_BAD_BLOCK_SIZE) { printf(" [FAIL] invalid code 99: expected %d, got %d\n", ZXC_ERROR_BAD_BLOCK_SIZE, rc); return 0; } printf(" [PASS] Invalid code 99 -> ZXC_ERROR_BAD_BLOCK_SIZE\n"); printf("PASS\n\n"); return 1; } zxc-0.11.0/tests/test_main.c000066400000000000000000000156151520102567100156750ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ #include "test_common.h" typedef int (*test_fn_t)(void); typedef struct { const char* name; test_fn_t fn; } test_entry_t; static const test_entry_t g_tests[] = { /* --- Streaming round-trip coverage (patterns x sizes x levels x checksum) --- */ TEST_CASE(test_roundtrip_raw_random), TEST_CASE(test_roundtrip_ghi_text), TEST_CASE(test_roundtrip_glo_text), TEST_CASE(test_roundtrip_num_seq), TEST_CASE(test_roundtrip_num_zero), TEST_CASE(test_roundtrip_num_small), TEST_CASE(test_roundtrip_num_large), TEST_CASE(test_roundtrip_small_50), TEST_CASE(test_roundtrip_empty), TEST_CASE(test_roundtrip_1byte), TEST_CASE(test_roundtrip_1byte_crc), TEST_CASE(test_roundtrip_large_15mb_lz), TEST_CASE(test_roundtrip_large_15mb_num), TEST_CASE(test_roundtrip_checksum_off), TEST_CASE(test_roundtrip_checksum_on), TEST_CASE(test_roundtrip_level1), TEST_CASE(test_roundtrip_level2), TEST_CASE(test_roundtrip_level3), TEST_CASE(test_roundtrip_level4), TEST_CASE(test_roundtrip_level5), TEST_CASE(test_roundtrip_level6), TEST_CASE(test_roundtrip_binary), TEST_CASE(test_roundtrip_binary_crc), TEST_CASE(test_roundtrip_binary_small), TEST_CASE(test_roundtrip_offset8_small), TEST_CASE(test_roundtrip_offset8_lvl5), TEST_CASE(test_roundtrip_offset16_large), TEST_CASE(test_roundtrip_offset16_lvl5), TEST_CASE(test_roundtrip_offset_mixed), /* --- Buffer API --- */ TEST_CASE(test_buffer_api), TEST_CASE(test_buffer_api_scratch_buf), TEST_CASE(test_buffer_error_codes), TEST_CASE(test_get_decompressed_size), TEST_CASE(test_decompress_fast_vs_safe_path), TEST_CASE(test_max_compressed_size_logic), /* --- Block API --- */ TEST_CASE(test_block_api), TEST_CASE(test_block_api_boundary_sizes), TEST_CASE(test_block_api_large_block_varint), TEST_CASE(test_decompress_block_bound), TEST_CASE(test_decompress_block_safe), /* --- Context API --- */ TEST_CASE(test_opaque_context_api), TEST_CASE(test_estimate_cctx_size), /* --- Stream API --- */ TEST_CASE(test_null_output_decompression), TEST_CASE(test_invalid_arguments), TEST_CASE(test_truncated_input), TEST_CASE(test_io_failures), TEST_CASE(test_thread_params), TEST_CASE(test_multithread_roundtrip), TEST_CASE(test_stream_get_decompressed_size_errors), TEST_CASE(test_stream_engine_errors), /* --- Push Streaming API --- */ TEST_CASE(test_pstream_roundtrip_basic), TEST_CASE(test_pstream_roundtrip_no_checksum), TEST_CASE(test_pstream_roundtrip_levels), TEST_CASE(test_pstream_tiny_chunks), TEST_CASE(test_pstream_drip_one_byte), TEST_CASE(test_pstream_empty_input), TEST_CASE(test_pstream_large_random), TEST_CASE(test_pstream_compatible_with_buffer_api), TEST_CASE(test_pstream_decompress_compatible_with_buffer_api), TEST_CASE(test_pstream_invalid_args), TEST_CASE(test_pstream_truncated_input), TEST_CASE(test_pstream_corrupted_magic), TEST_CASE(test_pstream_decode_seekable_archive), TEST_CASE(test_pstream_compress_after_end_rejected), TEST_CASE(test_pstream_compress_drain_block_resume), /* --- Format (on-disk) --- */ TEST_CASE(test_bit_reader), TEST_CASE(test_bitpack), TEST_CASE(test_huffman_codec), TEST_CASE(test_eof_block_structure), TEST_CASE(test_header_checksum), TEST_CASE(test_global_checksum_order), TEST_CASE(test_legacy_header), /* --- Misc --- */ TEST_CASE(test_error_name), TEST_CASE(test_library_info_api), /* --- Seekable (single-threaded) --- */ TEST_CASE(test_seekable_table_sizes), TEST_CASE(test_seekable_table_write), TEST_CASE(test_seekable_roundtrip), TEST_CASE(test_seekable_open_query), TEST_CASE(test_seekable_random_access), TEST_CASE(test_seekable_non_seekable_reject), TEST_CASE(test_seekable_single_block), TEST_CASE(test_seekable_all_levels), TEST_CASE(test_seekable_many_blocks), TEST_CASE(test_seekable_open_file), /* --- Seekable MT --- */ TEST_CASE(test_seekable_mt_roundtrip), TEST_CASE(test_seekable_mt_single_block), TEST_CASE(test_seekable_mt_random_access), TEST_CASE(test_seekable_mt_full_file), /* --- Seekable edge cases --- */ TEST_CASE(test_seekable_cross_boundary), TEST_CASE(test_seekable_truncated_input), TEST_CASE(test_seekable_corrupted_sek), TEST_CASE(test_seekable_range_out_of_bounds), TEST_CASE(test_seekable_dst_too_small), TEST_CASE(test_seekable_empty_file), TEST_CASE(test_seekable_no_checksum), TEST_CASE(test_seekable_with_checksum), }; static const size_t g_tests_count = sizeof(g_tests) / sizeof(g_tests[0]); static void print_usage(const char* argv0) { printf("Usage: %s [options] [filter]\n", argv0); printf(" filter substring matched against test names (e.g. '%s block_api')\n", argv0); printf(" -e, --exact N run only the test whose name exactly matches N\n"); printf(" --list print all test names and exit\n"); printf(" -h, --help show this help\n"); printf("With no argument, runs every test.\n"); } int main(int argc, char** argv) { srand(42); // Fixed seed for reproducibility const char* filter = NULL; int exact = 0; { int ai = 1; while (ai < argc) { const char* a = argv[ai]; if (strcmp(a, "-h") == 0 || strcmp(a, "--help") == 0) { print_usage(argv[0]); return 0; } if (strcmp(a, "--list") == 0) { for (size_t k = 0; k < g_tests_count; k++) printf("%s\n", g_tests[k].name); return 0; } if (strcmp(a, "-e") == 0 || strcmp(a, "--exact") == 0) { exact = 1; if (ai + 1 >= argc) { printf("error: %s requires a test name\n", a); return 1; } filter = argv[ai + 1]; ai += 2; continue; } filter = a; ai += 1; } } int total_failures = 0; int ran = 0; for (size_t i = 0; i < g_tests_count; i++) { if (filter) { const int match = exact ? (strcmp(g_tests[i].name, filter) == 0) : (strstr(g_tests[i].name, filter) != NULL); if (!match) continue; } if (!g_tests[i].fn()) total_failures++; ran++; } if (filter && ran == 0) { printf("No tests matched filter \"%s\"%s.\n", filter, exact ? " (exact)" : ""); return 1; } if (total_failures > 0) { printf("FAILED: %d tests failed.\n", total_failures); return 1; } printf("ALL TESTS PASSED SUCCESSFULLY.\n"); return 0; } zxc-0.11.0/tests/test_misc.c000066400000000000000000000074001520102567100156750ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ #include "test_common.h" int test_error_name() { printf("--- Test: zxc_error_name ---\n"); struct { int code; const char* expected; } cases[] = { {ZXC_OK, "ZXC_OK"}, {ZXC_ERROR_MEMORY, "ZXC_ERROR_MEMORY"}, {ZXC_ERROR_DST_TOO_SMALL, "ZXC_ERROR_DST_TOO_SMALL"}, {ZXC_ERROR_SRC_TOO_SMALL, "ZXC_ERROR_SRC_TOO_SMALL"}, {ZXC_ERROR_BAD_MAGIC, "ZXC_ERROR_BAD_MAGIC"}, {ZXC_ERROR_BAD_VERSION, "ZXC_ERROR_BAD_VERSION"}, {ZXC_ERROR_BAD_HEADER, "ZXC_ERROR_BAD_HEADER"}, {ZXC_ERROR_BAD_CHECKSUM, "ZXC_ERROR_BAD_CHECKSUM"}, {ZXC_ERROR_CORRUPT_DATA, "ZXC_ERROR_CORRUPT_DATA"}, {ZXC_ERROR_BAD_OFFSET, "ZXC_ERROR_BAD_OFFSET"}, {ZXC_ERROR_OVERFLOW, "ZXC_ERROR_OVERFLOW"}, {ZXC_ERROR_IO, "ZXC_ERROR_IO"}, {ZXC_ERROR_NULL_INPUT, "ZXC_ERROR_NULL_INPUT"}, {ZXC_ERROR_BAD_BLOCK_TYPE, "ZXC_ERROR_BAD_BLOCK_TYPE"}, {ZXC_ERROR_BAD_BLOCK_SIZE, "ZXC_ERROR_BAD_BLOCK_SIZE"}, }; const int n = sizeof(cases) / sizeof(cases[0]); for (int i = 0; i < n; i++) { const char* name = zxc_error_name(cases[i].code); if (strcmp(name, cases[i].expected) != 0) { printf(" [FAIL] zxc_error_name(%d) = \"%s\", expected \"%s\"\n", cases[i].code, name, cases[i].expected); return 0; } } printf(" [PASS] All %d known error codes\n", n); // Unknown codes should return "ZXC_UNKNOWN_ERROR" const char* unk = zxc_error_name(-999); if (strcmp(unk, "ZXC_UNKNOWN_ERROR") != 0) { printf(" [FAIL] zxc_error_name(-999) = \"%s\", expected \"ZXC_UNKNOWN_ERROR\"\n", unk); return 0; } unk = zxc_error_name(42); if (strcmp(unk, "ZXC_UNKNOWN_ERROR") != 0) { printf(" [FAIL] zxc_error_name(42) = \"%s\", expected \"ZXC_UNKNOWN_ERROR\"\n", unk); return 0; } printf(" [PASS] Unknown error codes\n"); printf("PASS\n\n"); return 1; } int test_library_info_api() { printf("=== TEST: Unit - Library Info API (zxc_min/max/default_level, zxc_version_string) ===\n"); // 1. Min level must match compile-time constant int min = zxc_min_level(); if (min != ZXC_LEVEL_FASTEST) { printf("Failed: zxc_min_level() returned %d, expected %d\n", min, ZXC_LEVEL_FASTEST); return 0; } printf(" [PASS] zxc_min_level() == %d\n", min); // 2. Max level must match compile-time constant int max = zxc_max_level(); if (max != ZXC_LEVEL_DENSITY) { printf("Failed: zxc_max_level() returned %d, expected %d\n", max, ZXC_LEVEL_DENSITY); return 0; } printf(" [PASS] zxc_max_level() == %d\n", max); // 3. Default level must be within [min, max] int def = zxc_default_level(); if (def < min || def > max) { printf("Failed: zxc_default_level() returned %d, not in [%d, %d]\n", def, min, max); return 0; } if (def != ZXC_LEVEL_DEFAULT) { printf("Failed: zxc_default_level() returned %d, expected %d\n", def, ZXC_LEVEL_DEFAULT); return 0; } printf(" [PASS] zxc_default_level() == %d\n", def); // 4. Version string must be non-NULL and match compile-time version const char* ver = zxc_version_string(); if (!ver) { printf("Failed: zxc_version_string() returned NULL\n"); return 0; } if (strcmp(ver, ZXC_LIB_VERSION_STR) != 0) { printf("Failed: zxc_version_string() returned \"%s\", expected \"%s\"\n", ver, ZXC_LIB_VERSION_STR); return 0; } printf(" [PASS] zxc_version_string() == \"%s\"\n", ver); printf("PASS\n\n"); return 1; } zxc-0.11.0/tests/test_pstream_api.c000066400000000000000000000601271520102567100172530ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause * * Tests for the push-based streaming API (zxc_pstream.h). * * Coverage focuses on: * - Round-trip correctness across patterns / sizes / chunk granularities. * - Wire-compatibility with the buffer API (push-compressed blob must be * decodable by zxc_decompress(), and vice-versa). * - Pathological caller behaviour (one-byte chunks, empty input, tiny * output buffers). * - Error handling on truncated / corrupted streams. */ #include "test_common.h" /* ---- helpers ---------------------------------------------------------- */ /* Push-compress src/src_size with given chunk sizes through the pstream * API and return a malloc'd blob (caller frees) of size *out_size. * Returns NULL on failure. */ static uint8_t* pstream_compress_in_chunks(const uint8_t* src, size_t src_size, size_t in_chunk, size_t out_chunk, int level, int checksum_enabled, size_t* out_size) { const zxc_compress_opts_t opts = {.level = level, .checksum_enabled = checksum_enabled}; zxc_cstream* cs = zxc_cstream_create(&opts); if (!cs) return NULL; size_t cap = 1024; size_t used = 0; uint8_t* blob = (uint8_t*)malloc(cap); uint8_t* obuf = (uint8_t*)malloc(out_chunk); if (!blob || !obuf) { free(blob); free(obuf); zxc_cstream_free(cs); return NULL; } /* Append helper */ #define APPEND_FROM(buf, n) \ do { \ if (used + (n) > cap) { \ while (used + (n) > cap) cap *= 2; \ uint8_t* nb = (uint8_t*)realloc(blob, cap); \ if (!nb) { \ free(blob); \ free(obuf); \ zxc_cstream_free(cs); \ return NULL; \ } \ blob = nb; \ } \ memcpy(blob + used, (buf), (n)); \ used += (n); \ } while (0) /* Phase 1: feed input in fixed chunks, draining as needed. */ size_t off = 0; zxc_outbuf_t out = {obuf, out_chunk, 0}; while (off < src_size) { const size_t n = (src_size - off) < in_chunk ? (src_size - off) : in_chunk; zxc_inbuf_t in = {src + off, n, 0}; while (in.pos < in.size) { const int64_t r = zxc_cstream_compress(cs, &out, &in); if (r < 0) { free(blob); free(obuf); zxc_cstream_free(cs); return NULL; } if (out.pos > 0) { APPEND_FROM(obuf, out.pos); out.pos = 0; } } off += n; } /* Phase 2: finalise. */ int64_t pending; do { pending = zxc_cstream_end(cs, &out); if (pending < 0) { free(blob); free(obuf); zxc_cstream_free(cs); return NULL; } if (out.pos > 0) { APPEND_FROM(obuf, out.pos); out.pos = 0; } } while (pending > 0); #undef APPEND_FROM free(obuf); zxc_cstream_free(cs); *out_size = used; return blob; } /* Push-decompress an entire blob in given chunk sizes; returns malloc'd * decoded buffer (caller frees) of size *out_size, or NULL on failure. */ static uint8_t* pstream_decompress_in_chunks(const uint8_t* src, size_t src_size, size_t in_chunk, size_t out_chunk, int checksum_enabled, size_t* out_size) { const zxc_decompress_opts_t opts = {.checksum_enabled = checksum_enabled}; zxc_dstream* ds = zxc_dstream_create(&opts); if (!ds) return NULL; size_t cap = 1024; size_t used = 0; uint8_t* dec = (uint8_t*)malloc(cap); uint8_t* obuf = (uint8_t*)malloc(out_chunk); if (!dec || !obuf) { free(dec); free(obuf); zxc_dstream_free(ds); return NULL; } /* Drive the loop until we have nothing more to feed *and* the decoder * makes no further progress (output drained, no input left to consume). */ size_t off = 0; zxc_outbuf_t out = {obuf, out_chunk, 0}; for (;;) { const size_t remaining = src_size - off; const size_t n = remaining < in_chunk ? remaining : in_chunk; zxc_inbuf_t in = {n ? src + off : NULL, n, 0}; const int64_t r = zxc_dstream_decompress(ds, &out, &in); if (r < 0) { free(dec); free(obuf); zxc_dstream_free(ds); return NULL; } if (out.pos > 0) { if (used + out.pos > cap) { while (used + out.pos > cap) cap *= 2; uint8_t* nb = (uint8_t*)realloc(dec, cap); if (!nb) { free(dec); free(obuf); zxc_dstream_free(ds); return NULL; } dec = nb; } memcpy(dec + used, obuf, out.pos); used += out.pos; out.pos = 0; } off += in.pos; /* Termination: no input left to feed and decoder produced nothing * AND consumed nothing -> it is either DONE or stalled. */ if (off >= src_size && in.pos == 0 && r == 0) break; } /* Reject truncated streams: if we ran out of input but the parser never * reached the validated footer, treat it as failure. */ if (!zxc_dstream_finished(ds)) { free(dec); free(obuf); zxc_dstream_free(ds); return NULL; } free(obuf); zxc_dstream_free(ds); *out_size = used; return dec; } /* End-to-end roundtrip with given chunk sizes; asserts the decoded blob * matches the input byte-for-byte. Returns 1 on success. */ static int do_roundtrip(const char* label, const uint8_t* src, size_t size, size_t in_chunk, size_t out_chunk, int level, int checksum_enabled) { size_t comp_size = 0; uint8_t* comp = pstream_compress_in_chunks(src, size, in_chunk, out_chunk, level, checksum_enabled, &comp_size); if (!comp) { printf(" [%s] compress failed\n", label); return 0; } size_t dec_size = 0; uint8_t* dec = pstream_decompress_in_chunks(comp, comp_size, in_chunk, out_chunk, checksum_enabled, &dec_size); if (!dec) { printf(" [%s] decompress failed (comp_size=%zu)\n", label, comp_size); free(comp); return 0; } if (dec_size != size || (size > 0 && memcmp(dec, src, size) != 0)) { printf(" [%s] mismatch (orig=%zu, dec=%zu)\n", label, size, dec_size); free(comp); free(dec); return 0; } free(comp); free(dec); return 1; } /* ---- tests ------------------------------------------------------------ */ int test_pstream_roundtrip_basic(void) { printf("=== TEST: PStream Roundtrip Basic (lz_data, 64 KiB) ===\n"); const size_t size = 64 * 1024; uint8_t* src = (uint8_t*)malloc(size); if (!src) return 0; gen_lz_data(src, size); const int ok = do_roundtrip("default 64KiB chunks", src, size, 64 * 1024, 64 * 1024, 3, 1); free(src); if (ok) printf("PASS\n\n"); return ok; } int test_pstream_roundtrip_no_checksum(void) { printf("=== TEST: PStream Roundtrip (checksum disabled) ===\n"); const size_t size = 80 * 1024; uint8_t* src = (uint8_t*)malloc(size); if (!src) return 0; gen_lz_data(src, size); const int ok = do_roundtrip("no csum", src, size, 32 * 1024, 32 * 1024, 3, 0); free(src); if (ok) printf("PASS\n\n"); return ok; } int test_pstream_roundtrip_levels(void) { printf("=== TEST: PStream Roundtrip across levels 1..5 ===\n"); const size_t size = 70 * 1024; uint8_t* src = (uint8_t*)malloc(size); if (!src) return 0; gen_lz_data(src, size); int ok = 1; char label[32]; for (int lvl = 1; lvl <= 5 && ok; lvl++) { snprintf(label, sizeof label, "level=%d csum=1", lvl); ok &= do_roundtrip(label, src, size, 16 * 1024, 16 * 1024, lvl, 1); } free(src); if (ok) printf("PASS\n\n"); return ok; } int test_pstream_tiny_chunks(void) { printf("=== TEST: PStream tiny IO chunks (in=137, out=53) ===\n"); /* Stresses the state machine with output buffers so small they force * partial drains in the middle of every block. */ const size_t size = 40 * 1024; uint8_t* src = (uint8_t*)malloc(size); if (!src) return 0; gen_lz_data(src, size); const int ok = do_roundtrip("tiny", src, size, 137, 53, 3, 1); free(src); if (ok) printf("PASS\n\n"); return ok; } int test_pstream_drip_one_byte(void) { printf("=== TEST: PStream 1-byte input chunks (worst-case feeder) ===\n"); const size_t size = 8 * 1024; uint8_t* src = (uint8_t*)malloc(size); if (!src) return 0; gen_lz_data(src, size); const int ok = do_roundtrip("1B", src, size, 1, 4096, 3, 1); free(src); if (ok) printf("PASS\n\n"); return ok; } int test_pstream_empty_input(void) { printf("=== TEST: PStream empty input -> valid empty file ===\n"); size_t comp_size = 0; uint8_t* comp = pstream_compress_in_chunks(NULL, 0, 4096, 4096, 3, 1, &comp_size); if (!comp) { printf("compress NULL/0 failed\n"); return 0; } /* Should at least be: file header (16) + EOF block (8) + footer (12) = 36 bytes. */ if (comp_size < 36) { printf("expected >=36 bytes, got %zu\n", comp_size); free(comp); return 0; } size_t dec_size = 0; uint8_t* dec = pstream_decompress_in_chunks(comp, comp_size, 4096, 4096, 1, &dec_size); if (!dec) { printf("decompress failed\n"); free(comp); return 0; } if (dec_size != 0) { printf("expected 0 decoded bytes, got %zu\n", dec_size); free(comp); free(dec); return 0; } free(comp); free(dec); printf("PASS (comp_size=%zu)\n\n", comp_size); return 1; } int test_pstream_large_random(void) { printf("=== TEST: PStream large mixed (1.5 MiB) ===\n"); const size_t size = 1500 * 1024; uint8_t* src = (uint8_t*)malloc(size); if (!src) return 0; gen_lz_data(src, size); const int ok = do_roundtrip("1.5MiB / level5", src, size, 13 * 1024, 7 * 1024, 5, 1); free(src); if (ok) printf("PASS\n\n"); return ok; } int test_pstream_compatible_with_buffer_api(void) { printf("=== TEST: pstream-compressed blob decodable by zxc_decompress() ===\n"); const size_t size = 100 * 1024; uint8_t* src = (uint8_t*)malloc(size); if (!src) return 0; gen_lz_data(src, size); size_t comp_size = 0; uint8_t* comp = pstream_compress_in_chunks(src, size, 8192, 8192, 3, 1, &comp_size); if (!comp) { free(src); return 0; } /* Decode with the one-shot buffer API. */ uint8_t* dec = (uint8_t*)malloc(size); if (!dec) { free(src); free(comp); return 0; } const zxc_decompress_opts_t dopts = {.checksum_enabled = 1}; const int64_t dsz = zxc_decompress(comp, comp_size, dec, size, &dopts); const int ok = (dsz == (int64_t)size && memcmp(dec, src, size) == 0); if (!ok) { printf("FAIL: dsz=%lld, expected %zu\n", (long long)dsz, size); } free(src); free(comp); free(dec); if (ok) printf("PASS\n\n"); return ok; } int test_pstream_decompress_compatible_with_buffer_api(void) { printf("=== TEST: buffer-API blob decodable by zxc_dstream ===\n"); const size_t size = 100 * 1024; uint8_t* src = (uint8_t*)malloc(size); if (!src) return 0; gen_lz_data(src, size); /* Compress with the one-shot buffer API. */ const uint64_t bound = zxc_compress_bound(size); uint8_t* comp = (uint8_t*)malloc((size_t)bound); if (!comp) { free(src); return 0; } const zxc_compress_opts_t copts = {.level = 3, .checksum_enabled = 1}; const int64_t csz = zxc_compress(src, size, comp, (size_t)bound, &copts); if (csz <= 0) { free(src); free(comp); return 0; } /* Decode with the streaming push API (small chunks). */ size_t dec_size = 0; uint8_t* dec = pstream_decompress_in_chunks(comp, (size_t)csz, 511, 7 * 1024, 1, &dec_size); const int ok = (dec && dec_size == size && memcmp(dec, src, size) == 0); if (!ok) { printf("FAIL: dec_size=%zu, expected %zu\n", dec_size, size); } free(src); free(comp); free(dec); if (ok) printf("PASS\n\n"); return ok; } int test_pstream_invalid_args(void) { printf("=== TEST: PStream invalid arguments ===\n"); /* NULL inputs must return ZXC_ERROR_NULL_INPUT, never crash. */ if (zxc_cstream_compress(NULL, NULL, NULL) != ZXC_ERROR_NULL_INPUT) return 0; if (zxc_cstream_end(NULL, NULL) != ZXC_ERROR_NULL_INPUT) return 0; if (zxc_dstream_decompress(NULL, NULL, NULL) != ZXC_ERROR_NULL_INPUT) return 0; if (zxc_cstream_in_size(NULL) != 0) return 0; if (zxc_cstream_out_size(NULL) != 0) return 0; if (zxc_dstream_in_size(NULL) != 0) return 0; if (zxc_dstream_out_size(NULL) != 0) return 0; /* Malformed descriptors must be rejected before any helper does an * unchecked memcpy() that could underflow size_t arithmetic or * dereference NULL. */ { const zxc_decompress_opts_t dopts = {0}; zxc_dstream* ds = zxc_dstream_create(&dopts); if (!ds) return 0; uint8_t buf[16] = {0}; zxc_outbuf_t good_out = {buf, sizeof buf, 0}; /* in->pos > in->size */ zxc_inbuf_t bad1 = {buf, 4, 5}; if (zxc_dstream_decompress(ds, &good_out, &bad1) != ZXC_ERROR_NULL_INPUT) { zxc_dstream_free(ds); return 0; } /* out->pos > out->size */ zxc_outbuf_t bad_out = {buf, 4, 5}; zxc_inbuf_t empty_in = {NULL, 0, 0}; if (zxc_dstream_decompress(ds, &bad_out, &empty_in) != ZXC_ERROR_NULL_INPUT) { zxc_dstream_free(ds); return 0; } /* in claims bytes but src is NULL */ zxc_inbuf_t bad2 = {NULL, 16, 0}; if (zxc_dstream_decompress(ds, &good_out, &bad2) != ZXC_ERROR_NULL_INPUT) { zxc_dstream_free(ds); return 0; } /* out claims capacity but dst is NULL */ zxc_outbuf_t bad_out2 = {NULL, 16, 0}; if (zxc_dstream_decompress(ds, &bad_out2, &empty_in) != ZXC_ERROR_NULL_INPUT) { zxc_dstream_free(ds); return 0; } zxc_dstream_free(ds); } /* free(NULL) is a no-op, must not crash. */ zxc_cstream_free(NULL); zxc_dstream_free(NULL); printf("PASS\n\n"); return 1; } int test_pstream_truncated_input(void) { printf("=== TEST: PStream truncated input -> error ===\n"); const size_t size = 64 * 1024; uint8_t* src = (uint8_t*)malloc(size); if (!src) return 0; gen_lz_data(src, size); size_t comp_size = 0; uint8_t* comp = pstream_compress_in_chunks(src, size, 4096, 4096, 3, 1, &comp_size); if (!comp || comp_size < 30) { free(src); free(comp); return 0; } /* Cut off the last 5 bytes (truncated footer). */ const size_t trunc_size = comp_size - 5; size_t dec_size = 0; uint8_t* dec = pstream_decompress_in_chunks(comp, trunc_size, 1024, 1024, 1, &dec_size); /* Either the helper returns NULL (error during decode), or it returns a * partial buffer that doesn't match, in both cases, NOT a clean success. */ const int ok = (dec == NULL) || (dec_size != size); free(src); free(comp); free(dec); if (ok) printf("PASS\n\n"); return ok; } int test_pstream_corrupted_magic(void) { printf("=== TEST: PStream corrupted file magic -> ZXC_ERROR_BAD_MAGIC ===\n"); /* Hand-craft 16 bogus bytes of "file header". */ uint8_t junk[16]; for (int i = 0; i < 16; i++) junk[i] = (uint8_t)(0xAA ^ i); const zxc_decompress_opts_t opts = {0}; zxc_dstream* ds = zxc_dstream_create(&opts); if (!ds) return 0; uint8_t out_buf[64]; zxc_outbuf_t out = {out_buf, sizeof out_buf, 0}; zxc_inbuf_t in = {junk, sizeof junk, 0}; const int64_t r = zxc_dstream_decompress(ds, &out, &in); int ok = (r == ZXC_ERROR_BAD_MAGIC); if (!ok) printf("FAIL: expected ZXC_ERROR_BAD_MAGIC (-4), got %lld\n", (long long)r); /* Sticky error: subsequent call returns the same code. */ zxc_inbuf_t empty = {NULL, 0, 0}; if (ok && zxc_dstream_decompress(ds, &out, &empty) != ZXC_ERROR_BAD_MAGIC) { printf("FAIL: error not sticky\n"); ok = 0; } zxc_dstream_free(ds); if (ok) printf("PASS\n\n"); return ok; } /* Decompress a SEEKABLE archive through the pstream API: after the EOF block * the decoder peeks 8 bytes, recognises a SEK block, and skips its payload * in DS_DRAIN_SEK_PAYLOAD before consuming the file footer. */ int test_pstream_decode_seekable_archive(void) { printf("=== TEST: PStream decodes seekable archive (DS_DRAIN_SEK_PAYLOAD) ===\n"); const size_t size = 96 * 1024; /* > one default block to force >1 SEK entry */ uint8_t* src = (uint8_t*)malloc(size); if (!src) return 0; gen_lz_data(src, size); /* Compress with seekable=1 via the buffer API (pstream cstream forces * seekable=0, so we need another producer to emit a SEK block). */ const uint64_t bound = zxc_compress_bound(size); uint8_t* comp = (uint8_t*)malloc((size_t)bound); if (!comp) { free(src); return 0; } const zxc_compress_opts_t copts = { .level = 3, .checksum_enabled = 1, .seekable = 1, .block_size = 32 * 1024}; const int64_t comp_size = zxc_compress(src, size, comp, (size_t)bound, &copts); if (comp_size <= 0) { printf("FAIL: zxc_compress(seekable=1) returned %lld\n", (long long)comp_size); free(src); free(comp); return 0; } /* Drive the seekable blob through zxc_dstream_decompress and check that * decoded bytes match the source. We feed the input in 4 KB chunks and * receive the output via 8 KB chunks: this stresses the SEK skip across * multiple calls (DS_DRAIN_SEK_PAYLOAD returns when the input is exhausted * mid-skip). */ const zxc_decompress_opts_t dopts = {.checksum_enabled = 1}; zxc_dstream* ds = zxc_dstream_create(&dopts); if (!ds) { free(src); free(comp); return 0; } uint8_t* dec = (uint8_t*)malloc(size); /* Zero-initialised: the decoder writes the first `out.pos` bytes per call * but cppcheck cannot see through the pointer chain in zxc_outbuf_t. */ uint8_t out_chunk[8192] = {0}; if (!dec) { zxc_dstream_free(ds); free(src); free(comp); return 0; } size_t dec_used = 0; size_t in_off = 0; const size_t in_step = 4096; int ok = 1; while (in_off < (size_t)comp_size && !zxc_dstream_finished(ds)) { const size_t n = ((size_t)comp_size - in_off) < in_step ? ((size_t)comp_size - in_off) : in_step; zxc_inbuf_t in = {comp + in_off, n, 0}; zxc_outbuf_t out = {out_chunk, sizeof out_chunk, 0}; const int64_t r = zxc_dstream_decompress(ds, &out, &in); if (r < 0) { printf("FAIL: zxc_dstream_decompress returned %lld\n", (long long)r); ok = 0; break; } if (dec_used + out.pos > size) { printf("FAIL: decoded bytes exceed source size\n"); ok = 0; break; } memcpy(dec + dec_used, out_chunk, out.pos); dec_used += out.pos; in_off += in.pos; if (in.pos == 0 && out.pos == 0) break; /* no progress */ } if (ok && !zxc_dstream_finished(ds)) { printf("FAIL: decoder did not finalise after consuming seekable input\n"); ok = 0; } if (ok && (dec_used != size || memcmp(dec, src, size) != 0)) { printf("FAIL: decoded payload does not match source\n"); ok = 0; } zxc_dstream_free(ds); free(src); free(comp); free(dec); if (ok) printf("PASS\n\n"); return ok; } /* Calling _compress() after _end() has started transitioning to a drain-tail * state must return ZXC_ERROR_NULL_INPUT. Covers the * `case CS_DRAIN_LAST/CS_DRAIN_EOF/CS_DRAIN_FOOTER/CS_ERRORED:` branch in * zxc_cstream_compress(). */ int test_pstream_compress_after_end_rejected(void) { printf("=== TEST: PStream _compress() after _end() -> ZXC_ERROR_NULL_INPUT ===\n"); const zxc_compress_opts_t opts = {.level = 3}; zxc_cstream* cs = zxc_cstream_create(&opts); if (!cs) return 0; /* Feed a few bytes so _end has actual residual data to flush. */ uint8_t src[64]; for (size_t i = 0; i < sizeof src; i++) src[i] = (uint8_t)i; uint8_t obuf[1024]; zxc_inbuf_t in = {src, sizeof src, 0}; zxc_outbuf_t out = {obuf, sizeof obuf, 0}; if (zxc_cstream_compress(cs, &out, &in) < 0) { zxc_cstream_free(cs); return 0; } /* Tiny output buffer so _end returns >0 (still pending) and the stream * is parked in CS_DRAIN_LAST / CS_DRAIN_EOF / CS_DRAIN_FOOTER. */ uint8_t tiny[4]; zxc_outbuf_t tiny_out = {tiny, sizeof tiny, 0}; const int64_t pending = zxc_cstream_end(cs, &tiny_out); if (pending <= 0) { printf("FAIL: expected _end() to return >0 with tiny output, got %lld\n", (long long)pending); zxc_cstream_free(cs); return 0; } /* Now state contains {CS_DRAIN_LAST, CS_DRAIN_EOF, CS_DRAIN_FOOTER}. _compress * must reject. */ zxc_inbuf_t more = {src, sizeof src, 0}; zxc_outbuf_t out2 = {obuf, sizeof obuf, 0}; const int64_t r = zxc_cstream_compress(cs, &out2, &more); const int ok = (r == ZXC_ERROR_NULL_INPUT); if (!ok) printf("FAIL: expected ZXC_ERROR_NULL_INPUT (-12), got %lld\n", (long long)r); zxc_cstream_free(cs); if (ok) printf("PASS\n\n"); return ok; } /* When _compress() fills a block but the caller's output buffer is too small * to drain it in one call, the next _compress() resumes at CS_DRAIN_BLOCK. * This test exercises that resume path inside zxc_cstream_compress (distinct * from the CS_DRAIN_BLOCK case in _end). */ int test_pstream_compress_drain_block_resume(void) { printf("=== TEST: PStream _compress() resumes CS_DRAIN_BLOCK across calls ===\n"); /* Small block size so we trigger the block boundary quickly. */ const zxc_compress_opts_t opts = {.level = 3, .block_size = 4096}; zxc_cstream* cs = zxc_cstream_create(&opts); if (!cs) return 0; /* Enough input for one full block (+ a little) but no more. */ const size_t src_size = 4096 + 256; uint8_t* src = (uint8_t*)malloc(src_size); if (!src) { zxc_cstream_free(cs); return 0; } gen_lz_data(src, src_size); /* Output buffer smaller than one compressed block: forces partial drains * and re-entries into CS_DRAIN_BLOCK. Zero-initialised to silence the * cppcheck false-positive that fires when an uninit array's address is * stored in zxc_outbuf_t.dst. */ uint8_t obuf[37] = {0}; zxc_inbuf_t in = {src, src_size, 0}; int saw_pending_drain = 0; int ok = 1; /* Drive the loop until input is exhausted; if any call returns >0 it * means the next call will hit CS_DRAIN_BLOCK in the switch. */ while (in.pos < in.size) { zxc_outbuf_t out = {obuf, sizeof obuf, 0}; const int64_t r = zxc_cstream_compress(cs, &out, &in); if (r < 0) { printf("FAIL: _compress returned %lld\n", (long long)r); ok = 0; break; } if (r > 0) saw_pending_drain = 1; } if (ok && !saw_pending_drain) { printf("FAIL: expected at least one >0 return to exercise CS_DRAIN_BLOCK resume\n"); ok = 0; } zxc_cstream_free(cs); free(src); if (ok) printf("PASS\n\n"); return ok; } zxc-0.11.0/tests/test_seekable.c000066400000000000000000001043501520102567100165170ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ #include "test_common.h" int test_seekable_table_sizes() { printf("=== TEST: Seekable - Table Sizes ===\n"); /* block_header(8) + 10*4 = 48 */ if (zxc_seek_table_size(10) != 48) { printf("Failed: size for 10 blocks\n"); return 0; } /* block_header(8) + 0 = 8 */ if (zxc_seek_table_size(0) != 8) { printf("Failed: zero blocks size\n"); return 0; } printf("PASS\n\n"); return 1; } int test_seekable_table_write() { printf("=== TEST: Seekable - Table Write/Validate ===\n"); const uint32_t comp[] = {100, 200, 150}; const size_t sz = zxc_seek_table_size(3); uint8_t* buf = malloc(sz); if (!buf) return 0; const int64_t written = zxc_write_seek_table(buf, sz, comp, 3); if (written != (int64_t)sz) { printf("Failed: write size mismatch\n"); free(buf); return 0; } /* Validate block_type == SEK in the block header */ if (buf[0] != ZXC_BLOCK_SEK) { printf("Failed: bad block_type (%u)\n", buf[0]); free(buf); return 0; } /* Validate first comp_size entry (after 8-byte block header) */ if (zxc_le32(buf + 8) != 100) { printf("Failed: bad first comp_size\n"); free(buf); return 0; } free(buf); printf("PASS\n\n"); return 1; } int test_seekable_roundtrip() { printf("=== TEST: Seekable - Compress/Decompress Roundtrip ===\n"); const size_t SRC_SIZE = 256 * 1024; uint8_t* src = malloc(SRC_SIZE); if (!src) return 0; fill_seek_data(src, SRC_SIZE, 42); const size_t dst_cap = (size_t)zxc_compress_bound(SRC_SIZE) + 1024; uint8_t* dst = malloc(dst_cap); uint8_t* dec = malloc(SRC_SIZE); if (!dst || !dec) { free(src); free(dst); free(dec); return 0; } zxc_compress_opts_t opts = {.level = ZXC_LEVEL_DEFAULT, .block_size = 64 * 1024, .checksum_enabled = 1, .seekable = 1}; const int64_t csize = zxc_compress(src, SRC_SIZE, dst, dst_cap, &opts); if (csize <= 0) { printf("Failed: compress\n"); free(src); free(dst); free(dec); return 0; } /* Sub-test A: single block (data < block_size) roundtrip */ { const size_t SMALL = 60 * 1024; /* fits in one 64KB block */ memset(dec, 0, SMALL); zxc_compress_opts_t opts_a = {.level = ZXC_LEVEL_DEFAULT, .block_size = 64 * 1024, .checksum_enabled = 1, .seekable = 0}; const int64_t a_csize = zxc_compress(src, SMALL, dst, dst_cap, &opts_a); if (a_csize <= 0) { printf("Failed: single-block 64KB compress (%lld)\n", (long long)a_csize); free(src); free(dst); free(dec); return 0; } zxc_decompress_opts_t ad = {.checksum_enabled = 1}; const int64_t a_dsize = zxc_decompress(dst, (size_t)a_csize, dec, SMALL, &ad); if (a_dsize != (int64_t)SMALL || memcmp(src, dec, SMALL) != 0) { printf("Failed: single-block 64KB roundtrip (dsize=%lld)\n", (long long)a_dsize); if (a_dsize == (int64_t)SMALL) { for (size_t i = 0; i < SMALL; i++) { if (src[i] != dec[i]) { printf(" first diff at byte %zu: src=0x%02x dec=0x%02x\n", i, src[i], dec[i]); break; } } } free(src); free(dst); free(dec); return 0; } printf(" sub-test A (single block, 60KB): OK\n"); } /* Sub-test B: exactly 2 blocks (128KB with 64KB block_size) */ { const size_t TWO = 128 * 1024; memset(dec, 0, TWO); zxc_compress_opts_t opts_b = {.level = ZXC_LEVEL_DEFAULT, .block_size = 64 * 1024, .checksum_enabled = 1, .seekable = 0}; const int64_t b_csize = zxc_compress(src, TWO, dst, dst_cap, &opts_b); if (b_csize <= 0) { printf("Failed: 2-block 64KB compress (%lld)\n", (long long)b_csize); free(src); free(dst); free(dec); return 0; } zxc_decompress_opts_t bd = {.checksum_enabled = 1}; const int64_t b_dsize = zxc_decompress(dst, (size_t)b_csize, dec, TWO, &bd); if (b_dsize != (int64_t)TWO || memcmp(src, dec, TWO) != 0) { printf("Failed: 2-block 64KB roundtrip (dsize=%lld)\n", (long long)b_dsize); if (b_dsize == (int64_t)TWO) { for (size_t i = 0; i < TWO; i++) { if (src[i] != dec[i]) { printf(" first diff at byte %zu: src=0x%02x dec=0x%02x\n", i, src[i], dec[i]); break; } } } free(src); free(dst); free(dec); return 0; } printf(" sub-test B (2 blocks, 128KB): OK\n"); } /* Sub-test C: full 256KB (4 blocks x 64KB) */ { memset(dec, 0, SRC_SIZE); zxc_compress_opts_t opts_ns = {.level = ZXC_LEVEL_DEFAULT, .block_size = 64 * 1024, .checksum_enabled = 1, .seekable = 0}; const int64_t ns_csize = zxc_compress(src, SRC_SIZE, dst, dst_cap, &opts_ns); if (ns_csize <= 0) { printf("Failed: non-seekable compress (%lld)\n", (long long)ns_csize); free(src); free(dst); free(dec); return 0; } zxc_decompress_opts_t nd = {.checksum_enabled = 1}; const int64_t ns_dsize = zxc_decompress(dst, (size_t)ns_csize, dec, SRC_SIZE, &nd); if (ns_dsize != (int64_t)SRC_SIZE || memcmp(src, dec, SRC_SIZE) != 0) { printf("Failed: non-seekable 64KB block_size roundtrip (dsize=%lld)\n", (long long)ns_dsize); if (ns_dsize == (int64_t)SRC_SIZE) { for (size_t i = 0; i < SRC_SIZE; i++) { if (src[i] != dec[i]) { printf(" first diff at byte %zu: src=0x%02x dec=0x%02x\n", i, src[i], dec[i]); break; } } } free(src); free(dst); free(dec); return 0; } printf(" sub-test C (4 blocks, 256KB): OK\n"); } /* Re-compress with seekable=1 for the actual test */ memset(dec, 0, SRC_SIZE); const int64_t csize2 = zxc_compress(src, SRC_SIZE, dst, dst_cap, &opts); if (csize2 <= 0) { printf("Failed: seekable re-compress (%lld)\n", (long long)csize2); free(src); free(dst); free(dec); return 0; } /* Full decompression with standard API (backward compatibility) */ zxc_decompress_opts_t dopts = {.checksum_enabled = 1}; const int64_t dsize = zxc_decompress(dst, (size_t)csize2, dec, SRC_SIZE, &dopts); if (dsize != (int64_t)SRC_SIZE || memcmp(src, dec, SRC_SIZE) != 0) { printf("Failed: decompress mismatch (csize=%lld dsize=%lld expected=%zu)\n", (long long)csize2, (long long)dsize, SRC_SIZE); if (dsize == (int64_t)SRC_SIZE) { for (size_t i = 0; i < SRC_SIZE; i++) { if (src[i] != dec[i]) { printf(" first diff at byte %zu: src=0x%02x dec=0x%02x\n", i, src[i], dec[i]); break; } } } free(src); free(dst); free(dec); return 0; } free(src); free(dst); free(dec); printf("PASS\n\n"); return 1; } int test_seekable_open_query() { printf("=== TEST: Seekable - Open and Query ===\n"); const size_t SRC_SIZE = 200 * 1024; uint8_t* src = malloc(SRC_SIZE); if (!src) return 0; fill_seek_data(src, SRC_SIZE, 99); const size_t dst_cap = (size_t)zxc_compress_bound(SRC_SIZE) + 1024; uint8_t* dst = malloc(dst_cap); if (!dst) { free(src); return 0; } zxc_compress_opts_t opts = {.level = 2, .block_size = 64 * 1024, .seekable = 1}; const int64_t csize = zxc_compress(src, SRC_SIZE, dst, dst_cap, &opts); if (csize <= 0) { printf("Failed: compress\n"); free(src); free(dst); return 0; } zxc_seekable* s = zxc_seekable_open(dst, (size_t)csize); if (!s) { printf("Failed: open\n"); free(src); free(dst); return 0; } const uint32_t nb = zxc_seekable_get_num_blocks(s); if (nb < 3) { printf("Failed: expected >= 3 blocks, got %u\n", nb); zxc_seekable_free(s); free(src); free(dst); return 0; } const uint64_t total = zxc_seekable_get_decompressed_size(s); if (total != SRC_SIZE) { printf("Failed: decomp size\n"); zxc_seekable_free(s); free(src); free(dst); return 0; } uint64_t sum = 0; for (uint32_t i = 0; i < nb; i++) { sum += zxc_seekable_get_block_decomp_size(s, i); if (zxc_seekable_get_block_comp_size(s, i) == 0) { printf("Failed: zero comp size\n"); zxc_seekable_free(s); free(src); free(dst); return 0; } } if (sum != SRC_SIZE) { printf("Failed: block sizes sum\n"); zxc_seekable_free(s); free(src); free(dst); return 0; } zxc_seekable_free(s); free(src); free(dst); printf("PASS\n\n"); return 1; } int test_seekable_random_access() { printf("=== TEST: Seekable - Random Access ===\n"); const size_t SRC_SIZE = 300 * 1024; uint8_t* src = malloc(SRC_SIZE); if (!src) return 0; fill_seek_data(src, SRC_SIZE, 77); const size_t dst_cap = (size_t)zxc_compress_bound(SRC_SIZE) + 1024; uint8_t* dst = malloc(dst_cap); if (!dst) { free(src); return 0; } zxc_compress_opts_t opts = {.level = 3, .block_size = 64 * 1024, .seekable = 1}; const int64_t csize = zxc_compress(src, SRC_SIZE, dst, dst_cap, &opts); if (csize <= 0) { printf("Failed: compress\n"); free(src); free(dst); return 0; } zxc_seekable* s = zxc_seekable_open(dst, (size_t)csize); if (!s) { printf("Failed: open\n"); free(src); free(dst); return 0; } /* First 1000 bytes */ uint8_t out1[1000]; int64_t r = zxc_seekable_decompress_range(s, out1, 1000, 0, 1000); if (r != 1000 || memcmp(src, out1, 1000) != 0) { printf("Failed: first 1000 bytes (r=%lld)\n", (long long)r); if (r == 1000) { for (int i = 0; i < 1000; i++) { if (src[i] != out1[i]) { printf(" first diff at byte %d: src=0x%02x out=0x%02x\n", i, src[i], out1[i]); break; } } } zxc_seekable_free(s); free(src); free(dst); return 0; } /* Middle range spanning multiple blocks */ const uint64_t off = 100 * 1024; const size_t len = 80 * 1024; uint8_t* out2 = malloc(len); if (!out2) { zxc_seekable_free(s); free(src); free(dst); return 0; } r = zxc_seekable_decompress_range(s, out2, len, off, len); if (r != (int64_t)len || memcmp(src + off, out2, len) != 0) { printf("Failed: mid-range\n"); free(out2); zxc_seekable_free(s); free(src); free(dst); return 0; } free(out2); /* Last bytes */ uint8_t out3[512]; r = zxc_seekable_decompress_range(s, out3, 512, SRC_SIZE - 512, 512); if (r != 512 || memcmp(src + SRC_SIZE - 512, out3, 512) != 0) { printf("Failed: tail\n"); zxc_seekable_free(s); free(src); free(dst); return 0; } /* Entire file */ uint8_t* out4 = malloc(SRC_SIZE); if (!out4) { zxc_seekable_free(s); free(src); free(dst); return 0; } r = zxc_seekable_decompress_range(s, out4, SRC_SIZE, 0, SRC_SIZE); if (r != (int64_t)SRC_SIZE || memcmp(src, out4, SRC_SIZE) != 0) { printf("Failed: full range\n"); free(out4); zxc_seekable_free(s); free(src); free(dst); return 0; } free(out4); /* Zero length */ uint8_t dummy; r = zxc_seekable_decompress_range(s, &dummy, 1, 0, 0); if (r != 0) { printf("Failed: zero-length\n"); zxc_seekable_free(s); free(src); free(dst); return 0; } zxc_seekable_free(s); free(src); free(dst); printf("PASS\n\n"); return 1; } int test_seekable_non_seekable_reject() { printf("=== TEST: Seekable - Non-Seekable Archive Rejected ===\n"); const size_t SRC_SIZE = 10000; uint8_t* src = malloc(SRC_SIZE); if (!src) return 0; fill_seek_data(src, SRC_SIZE, 11); const size_t dst_cap = (size_t)zxc_compress_bound(SRC_SIZE); uint8_t* dst = malloc(dst_cap); if (!dst) { free(src); return 0; } zxc_compress_opts_t opts = {.level = 1, .seekable = 0}; const int64_t csize = zxc_compress(src, SRC_SIZE, dst, dst_cap, &opts); if (csize <= 0) { printf("Failed: compress\n"); free(src); free(dst); return 0; } zxc_seekable* s = zxc_seekable_open(dst, (size_t)csize); if (s != NULL) { printf("Failed: expected NULL for non-seekable\n"); zxc_seekable_free(s); free(src); free(dst); return 0; } free(src); free(dst); printf("PASS\n\n"); return 1; } int test_seekable_single_block() { printf("=== TEST: Seekable - Single Block ===\n"); const size_t SRC_SIZE = 1024; uint8_t* src = malloc(SRC_SIZE); if (!src) return 0; fill_seek_data(src, SRC_SIZE, 55); const size_t dst_cap = (size_t)zxc_compress_bound(SRC_SIZE) + 256; uint8_t* dst = malloc(dst_cap); if (!dst) { free(src); return 0; } zxc_compress_opts_t opts = {.level = 1, .seekable = 1}; const int64_t csize = zxc_compress(src, SRC_SIZE, dst, dst_cap, &opts); if (csize <= 0) { printf("Failed: compress\n"); free(src); free(dst); return 0; } zxc_seekable* s = zxc_seekable_open(dst, (size_t)csize); if (!s) { printf("Failed: open\n"); free(src); free(dst); return 0; } if (zxc_seekable_get_num_blocks(s) != 1) { printf("Failed: expected 1 block\n"); zxc_seekable_free(s); free(src); free(dst); return 0; } uint8_t out[100]; int64_t r = zxc_seekable_decompress_range(s, out, 100, 500, 100); if (r != 100 || memcmp(src + 500, out, 100) != 0) { printf("Failed: range data\n"); zxc_seekable_free(s); free(src); free(dst); return 0; } zxc_seekable_free(s); free(src); free(dst); printf("PASS\n\n"); return 1; } int test_seekable_all_levels() { printf("=== TEST: Seekable - All Compression Levels ===\n"); const size_t SRC_SIZE = 128 * 1024; uint8_t* src = malloc(SRC_SIZE); if (!src) return 0; fill_seek_data(src, SRC_SIZE, 33); const size_t dst_cap = (size_t)zxc_compress_bound(SRC_SIZE) + 1024; uint8_t* dst = malloc(dst_cap); uint8_t* dec = malloc(SRC_SIZE); if (!dst || !dec) { free(src); free(dst); free(dec); return 0; } for (int lvl = 1; lvl <= 5; lvl++) { zxc_compress_opts_t opts = {.level = lvl, .block_size = 32 * 1024, .seekable = 1}; const int64_t csize = zxc_compress(src, SRC_SIZE, dst, dst_cap, &opts); if (csize <= 0) { printf("Failed: compress level %d\n", lvl); free(src); free(dst); free(dec); return 0; } zxc_seekable* s = zxc_seekable_open(dst, (size_t)csize); if (!s) { printf("Failed: open level %d\n", lvl); free(src); free(dst); free(dec); return 0; } int64_t r = zxc_seekable_decompress_range(s, dec, SRC_SIZE, 0, SRC_SIZE); if (r != (int64_t)SRC_SIZE || memcmp(src, dec, SRC_SIZE) != 0) { printf("Failed: level %d data mismatch (r=%lld expected=%zu)\n", lvl, (long long)r, SRC_SIZE); if (r == (int64_t)SRC_SIZE) { for (size_t i = 0; i < SRC_SIZE; i++) { if (src[i] != dec[i]) { printf(" first diff at byte %zu: src=0x%02x dec=0x%02x\n", i, src[i], dec[i]); break; } } } zxc_seekable_free(s); free(src); free(dst); free(dec); return 0; } zxc_seekable_free(s); } free(src); free(dst); free(dec); printf("PASS\n\n"); return 1; } int test_seekable_many_blocks() { printf("=== TEST: Seekable - Many Small Blocks ===\n"); /* Use minimum block size (4KB) with 256KB data => 64 blocks. * This stresses the seekable block tracking array (dispatch lines 410-424) * and ensures the seek table handles high block counts correctly. */ const size_t SRC_SIZE = 256 * 1024; uint8_t* src = malloc(SRC_SIZE); if (!src) return 0; fill_seek_data(src, SRC_SIZE, 77); const size_t dst_cap = (size_t)zxc_compress_bound(SRC_SIZE) + 4096; uint8_t* dst = malloc(dst_cap); uint8_t* dec = malloc(SRC_SIZE); if (!dst || !dec) { free(src); free(dst); free(dec); return 0; } /* Compress with minimum block_size = 4096 */ zxc_compress_opts_t opts = {.level = ZXC_LEVEL_DEFAULT, .block_size = 4096, .checksum_enabled = 1, .seekable = 1}; const int64_t csize = zxc_compress(src, SRC_SIZE, dst, dst_cap, &opts); if (csize <= 0) { printf("Failed: compress (%lld)\n", (long long)csize); free(src); free(dst); free(dec); return 0; } /* Open and verify block count = 64 */ zxc_seekable* s = zxc_seekable_open(dst, (size_t)csize); if (!s) { printf("Failed: open\n"); free(src); free(dst); free(dec); return 0; } const uint32_t n_blocks = zxc_seekable_get_num_blocks(s); if (n_blocks != 64) { printf("Failed: expected 64 blocks, got %u\n", n_blocks); zxc_seekable_free(s); free(src); free(dst); free(dec); return 0; } /* Full decompress via seekable API */ int64_t r = zxc_seekable_decompress_range(s, dec, SRC_SIZE, 0, SRC_SIZE); if (r != (int64_t)SRC_SIZE || memcmp(src, dec, SRC_SIZE) != 0) { printf("Failed: full decompress mismatch (r=%lld)\n", (long long)r); zxc_seekable_free(s); free(src); free(dst); free(dec); return 0; } /* Random access: read 100 bytes from the middle of block 32 */ const uint64_t mid_off = 32 * 4096 + 2000; uint8_t spot[100]; r = zxc_seekable_decompress_range(s, spot, 100, mid_off, 100); if (r != 100 || memcmp(src + mid_off, spot, 100) != 0) { printf("Failed: random access at offset %llu\n", (unsigned long long)mid_off); zxc_seekable_free(s); free(src); free(dst); free(dec); return 0; } /* Cross-block read: span block boundary (last 50B of block 15 + first 50B of block 16) */ const uint64_t cross_off = 16 * 4096 - 50; uint8_t cross[100]; r = zxc_seekable_decompress_range(s, cross, 100, cross_off, 100); if (r != 100 || memcmp(src + cross_off, cross, 100) != 0) { printf("Failed: cross-block read at offset %llu\n", (unsigned long long)cross_off); zxc_seekable_free(s); free(src); free(dst); free(dec); return 0; } zxc_seekable_free(s); free(src); free(dst); free(dec); printf("PASS\n\n"); return 1; } int test_seekable_open_file() { printf("=== TEST: Seekable - Open File ===\n"); /* Compress seekable data into a buffer, write to tmpfile, then open via file API */ const size_t SRC_SIZE = 128 * 1024; uint8_t* src = malloc(SRC_SIZE); if (!src) return 0; fill_seek_data(src, SRC_SIZE, 99); const size_t dst_cap = (size_t)zxc_compress_bound(SRC_SIZE) + 1024; uint8_t* dst = malloc(dst_cap); uint8_t* dec = malloc(SRC_SIZE); if (!dst || !dec) { free(src); free(dst); free(dec); return 0; } zxc_compress_opts_t opts = {.level = ZXC_LEVEL_DEFAULT, .block_size = 32 * 1024, .checksum_enabled = 1, .seekable = 1}; const int64_t csize = zxc_compress(src, SRC_SIZE, dst, dst_cap, &opts); if (csize <= 0) { printf("Failed: compress (%lld)\n", (long long)csize); free(src); free(dst); free(dec); return 0; } /* Write compressed data to a temp file */ FILE* tf = tmpfile(); if (!tf) { printf("Failed: tmpfile\n"); free(src); free(dst); free(dec); return 0; } if (fwrite(dst, 1, (size_t)csize, tf) != (size_t)csize) { printf("Failed: fwrite\n"); fclose(tf); free(src); free(dst); free(dec); return 0; } fflush(tf); /* Open via file API */ zxc_seekable* s = zxc_seekable_open_file(tf); if (!s) { printf("Failed: zxc_seekable_open_file returned NULL\n"); fclose(tf); free(src); free(dst); free(dec); return 0; } /* Verify block count: 128KB / 32KB = 4 blocks */ const uint32_t n_blocks = zxc_seekable_get_num_blocks(s); if (n_blocks != 4) { printf("Failed: expected 4 blocks, got %u\n", n_blocks); zxc_seekable_free(s); fclose(tf); free(src); free(dst); free(dec); return 0; } /* Full decompress from file */ int64_t r = zxc_seekable_decompress_range(s, dec, SRC_SIZE, 0, SRC_SIZE); if (r != (int64_t)SRC_SIZE || memcmp(src, dec, SRC_SIZE) != 0) { printf("Failed: full decompress from file (r=%lld)\n", (long long)r); zxc_seekable_free(s); fclose(tf); free(src); free(dst); free(dec); return 0; } /* Random access from file: read 200 bytes spanning block boundary */ const uint64_t cross_off = 32 * 1024 - 100; /* last 100B of block 0 + first 100B of block 1 */ uint8_t cross[200]; r = zxc_seekable_decompress_range(s, cross, 200, cross_off, 200); if (r != 200 || memcmp(src + cross_off, cross, 200) != 0) { printf("Failed: cross-block read from file\n"); zxc_seekable_free(s); fclose(tf); free(src); free(dst); free(dec); return 0; } zxc_seekable_free(s); fclose(tf); /* NULL input rejection */ if (zxc_seekable_open_file(NULL) != NULL) { printf("Failed: NULL not rejected\n"); free(src); free(dst); free(dec); return 0; } free(src); free(dst); free(dec); printf("PASS\n\n"); return 1; } /* Cross-boundary range: decompresses bytes that span exactly two blocks */ int test_seekable_cross_boundary() { printf("=== TEST: Seekable - Cross-Boundary Range ===\n"); const size_t BLK = 64 * 1024; const size_t SRC_SIZE = BLK * 4; /* 4 blocks */ uint8_t* src = malloc(SRC_SIZE); if (!src) return 0; fill_seek_data(src, SRC_SIZE, 123); const size_t dst_cap = (size_t)zxc_compress_bound(SRC_SIZE) + 1024; uint8_t* dst = malloc(dst_cap); if (!dst) { free(src); return 0; } zxc_compress_opts_t opts = {.level = 3, .block_size = BLK, .seekable = 1}; const int64_t csize = zxc_compress(src, SRC_SIZE, dst, dst_cap, &opts); if (csize <= 0) { printf("Failed: compress\n"); free(src); free(dst); return 0; } zxc_seekable* s = zxc_seekable_open(dst, (size_t)csize); if (!s) { printf("Failed: open\n"); free(src); free(dst); return 0; } /* Read 200 bytes starting 100 bytes before the block 0/1 boundary */ const uint64_t boundary = BLK; const uint64_t off = boundary - 100; const size_t len = 200; uint8_t out[200]; int64_t r = zxc_seekable_decompress_range(s, out, sizeof(out), off, len); if (r != (int64_t)len) { printf("Failed: cross-boundary range returned %lld\n", (long long)r); zxc_seekable_free(s); free(src); free(dst); return 0; } if (memcmp(out, src + off, len) != 0) { printf("Failed: cross-boundary data mismatch\n"); zxc_seekable_free(s); free(src); free(dst); return 0; } /* Also test a range spanning 3 blocks */ const uint64_t off3 = BLK * 2 - 100; const size_t len3 = BLK + 200; uint8_t* out3 = malloc(len3); if (!out3) { zxc_seekable_free(s); free(src); free(dst); return 0; } r = zxc_seekable_decompress_range(s, out3, len3, off3, len3); if (r != (int64_t)len3 || memcmp(out3, src + off3, len3) != 0) { printf("Failed: 3-block span mismatch\n"); free(out3); zxc_seekable_free(s); free(src); free(dst); return 0; } free(out3); zxc_seekable_free(s); free(src); free(dst); printf("PASS\n\n"); return 1; } /* Open with truncated data should return NULL */ int test_seekable_truncated_input() { printf("=== TEST: Seekable - Truncated Input Rejected ===\n"); const size_t SRC_SIZE = 64 * 1024; uint8_t* src = malloc(SRC_SIZE); if (!src) return 0; fill_seek_data(src, SRC_SIZE, 44); const size_t dst_cap = (size_t)zxc_compress_bound(SRC_SIZE) + 256; uint8_t* dst = malloc(dst_cap); if (!dst) { free(src); return 0; } zxc_compress_opts_t opts = {.level = 1, .seekable = 1}; const int64_t csize = zxc_compress(src, SRC_SIZE, dst, dst_cap, &opts); if (csize <= 0) { printf("Failed: compress\n"); free(src); free(dst); return 0; } /* Truncate to half */ zxc_seekable* s = zxc_seekable_open(dst, (size_t)(csize / 2)); if (s != NULL) { printf("Failed: should reject truncated data\n"); zxc_seekable_free(s); free(src); free(dst); return 0; } /* Truncate to just header */ s = zxc_seekable_open(dst, 16); if (s != NULL) { printf("Failed: should reject header-only data\n"); zxc_seekable_free(s); free(src); free(dst); return 0; } /* Zero bytes */ s = zxc_seekable_open(dst, 0); if (s != NULL) { printf("Failed: should reject zero-length data\n"); zxc_seekable_free(s); free(src); free(dst); return 0; } free(src); free(dst); printf("PASS\n\n"); return 1; } /* Corrupted SEK block: ensure no crash (no UB) */ int test_seekable_corrupted_sek() { printf("=== TEST: Seekable - Corrupted SEK Block ===\n"); const size_t SRC_SIZE = 64 * 1024; uint8_t* src = malloc(SRC_SIZE); if (!src) return 0; fill_seek_data(src, SRC_SIZE, 66); const size_t dst_cap = (size_t)zxc_compress_bound(SRC_SIZE) + 256; uint8_t* dst = malloc(dst_cap); if (!dst) { free(src); return 0; } zxc_compress_opts_t opts = {.level = 1, .seekable = 1}; const int64_t csize = zxc_compress(src, SRC_SIZE, dst, dst_cap, &opts); if (csize <= 0) { printf("Failed: compress\n"); free(src); free(dst); return 0; } /* Corrupt a byte in the SEK payload area (before footer) */ uint8_t* corrupt = malloc((size_t)csize); if (!corrupt) { free(src); free(dst); return 0; } memcpy(corrupt, dst, (size_t)csize); corrupt[csize - 14] ^= 0xFF; zxc_seekable* s = zxc_seekable_open(corrupt, (size_t)csize); /* May succeed or fail - just ensure no crash */ if (s) { uint8_t out[100]; (void)zxc_seekable_decompress_range(s, out, sizeof(out), 0, 100); zxc_seekable_free(s); } free(corrupt); free(src); free(dst); printf("PASS\n\n"); return 1; } /* Range beyond file end should return error */ int test_seekable_range_out_of_bounds() { printf("=== TEST: Seekable - Out-of-Bounds Range ===\n"); const size_t SRC_SIZE = 32 * 1024; uint8_t* src = malloc(SRC_SIZE); if (!src) return 0; fill_seek_data(src, SRC_SIZE, 22); const size_t dst_cap = (size_t)zxc_compress_bound(SRC_SIZE) + 256; uint8_t* dst = malloc(dst_cap); if (!dst) { free(src); return 0; } zxc_compress_opts_t opts = {.level = 1, .seekable = 1}; const int64_t csize = zxc_compress(src, SRC_SIZE, dst, dst_cap, &opts); if (csize <= 0) { printf("Failed: compress\n"); free(src); free(dst); return 0; } zxc_seekable* s = zxc_seekable_open(dst, (size_t)csize); if (!s) { printf("Failed: open\n"); free(src); free(dst); return 0; } uint8_t out[256]; /* offset past EOF */ int64_t r = zxc_seekable_decompress_range(s, out, sizeof(out), SRC_SIZE + 100, 100); if (r > 0) { printf("Failed: should reject offset past EOF (got %lld)\n", (long long)r); zxc_seekable_free(s); free(src); free(dst); return 0; } /* offset valid but length extends past EOF */ r = zxc_seekable_decompress_range(s, out, sizeof(out), SRC_SIZE - 50, 200); if (r > 0) { printf("Failed: should reject range extending past EOF (got %lld)\n", (long long)r); zxc_seekable_free(s); free(src); free(dst); return 0; } zxc_seekable_free(s); free(src); free(dst); printf("PASS\n\n"); return 1; } /* dst_capacity too small for requested range */ int test_seekable_dst_too_small() { printf("=== TEST: Seekable - Dst Too Small ===\n"); const size_t SRC_SIZE = 32 * 1024; uint8_t* src = malloc(SRC_SIZE); if (!src) return 0; fill_seek_data(src, SRC_SIZE, 91); const size_t dst_cap = (size_t)zxc_compress_bound(SRC_SIZE) + 256; uint8_t* dst = malloc(dst_cap); if (!dst) { free(src); return 0; } zxc_compress_opts_t opts = {.level = 1, .seekable = 1}; const int64_t csize = zxc_compress(src, SRC_SIZE, dst, dst_cap, &opts); if (csize <= 0) { printf("Failed: compress\n"); free(src); free(dst); return 0; } zxc_seekable* s = zxc_seekable_open(dst, (size_t)csize); if (!s) { printf("Failed: open\n"); free(src); free(dst); return 0; } uint8_t out[10]; int64_t r = zxc_seekable_decompress_range(s, out, 10, 0, 1000); if (r > 0) { printf("Failed: should reject insufficient dst capacity (got %lld)\n", (long long)r); zxc_seekable_free(s); free(src); free(dst); return 0; } zxc_seekable_free(s); free(src); free(dst); printf("PASS\n\n"); return 1; } /* Empty file with seekable=1 */ /* Empty file with seekable=1: buffer API rejects NULL src, verify graceful rejection. * Also verify via streaming API (which supports empty files). */ int test_seekable_empty_file() { printf("=== TEST: Seekable - Empty File ===\n"); const size_t dst_cap = (size_t)zxc_compress_bound(0) + 256; uint8_t* dst = malloc(dst_cap); if (!dst) return 0; /* Buffer API: NULL src with size 0 is rejected with ZXC_ERROR_NULL_INPUT */ zxc_compress_opts_t opts = {.level = 3, .seekable = 1}; const int64_t csize = zxc_compress(NULL, 0, dst, dst_cap, &opts); if (csize >= 0) { printf("Failed: expected NULL_INPUT rejection (got %lld)\n", (long long)csize); free(dst); return 0; } /* Streaming API: empty file via tmpfile() should work */ FILE* fin = tmpfile(); FILE* fout = tmpfile(); if (fin && fout) { int64_t stream_sz = zxc_stream_compress(fin, fout, &opts); if (stream_sz < 0) { printf("Failed: stream compress empty (got %lld)\n", (long long)stream_sz); fclose(fin); fclose(fout); free(dst); return 0; } /* Decompress the stream output */ rewind(fout); FILE* fdec = tmpfile(); if (fdec) { zxc_decompress_opts_t dopts = {.checksum_enabled = 0}; int64_t dsz = zxc_stream_decompress(fout, fdec, &dopts); if (dsz != 0) { printf("Failed: stream decompress empty should return 0 (got %lld)\n", (long long)dsz); fclose(fin); fclose(fout); fclose(fdec); free(dst); return 0; } fclose(fdec); } fclose(fin); fclose(fout); } free(dst); printf("PASS\n\n"); return 1; } /* Seekable without checksum (seekable=1, checksum_enabled=0) */ int test_seekable_no_checksum() { printf("=== TEST: Seekable - No Checksum ===\n"); const size_t SRC_SIZE = 256 * 1024; uint8_t* src = malloc(SRC_SIZE); if (!src) return 0; fill_seek_data(src, SRC_SIZE, 31); const size_t dst_cap = (size_t)zxc_compress_bound(SRC_SIZE) + 1024; uint8_t* dst = malloc(dst_cap); if (!dst) { free(src); return 0; } zxc_compress_opts_t opts = { .level = 3, .block_size = 64 * 1024, .seekable = 1, .checksum_enabled = 0}; const int64_t csize = zxc_compress(src, SRC_SIZE, dst, dst_cap, &opts); if (csize <= 0) { printf("Failed: compress\n"); free(src); free(dst); return 0; } zxc_seekable* s = zxc_seekable_open(dst, (size_t)csize); if (!s) { printf("Failed: open\n"); free(src); free(dst); return 0; } uint8_t out[512]; const uint64_t off = 64 * 1024 + 100; int64_t r = zxc_seekable_decompress_range(s, out, sizeof(out), off, 512); if (r != 512 || memcmp(out, src + off, 512) != 0) { printf("Failed: no-checksum range mismatch\n"); zxc_seekable_free(s); free(src); free(dst); return 0; } /* Full decompress also works */ uint8_t* full = malloc(SRC_SIZE); if (!full) { zxc_seekable_free(s); free(src); free(dst); return 0; } zxc_decompress_opts_t dopts = {.checksum_enabled = 0}; int64_t dsize = zxc_decompress(dst, (size_t)csize, full, SRC_SIZE, &dopts); if (dsize != (int64_t)SRC_SIZE || memcmp(src, full, SRC_SIZE) != 0) { printf("Failed: full decompress mismatch\n"); free(full); zxc_seekable_free(s); free(src); free(dst); return 0; } free(full); zxc_seekable_free(s); free(src); free(dst); printf("PASS\n\n"); return 1; } /* Seekable with checksum (seekable=1, checksum_enabled=1) */ int test_seekable_with_checksum() { printf("=== TEST: Seekable - With Checksum ===\n"); const size_t SRC_SIZE = 256 * 1024; uint8_t* src = malloc(SRC_SIZE); if (!src) return 0; fill_seek_data(src, SRC_SIZE, 47); const size_t dst_cap = (size_t)zxc_compress_bound(SRC_SIZE) + 1024; uint8_t* dst = malloc(dst_cap); if (!dst) { free(src); return 0; } zxc_compress_opts_t opts = { .level = 3, .block_size = 64 * 1024, .seekable = 1, .checksum_enabled = 1}; const int64_t csize = zxc_compress(src, SRC_SIZE, dst, dst_cap, &opts); if (csize <= 0) { printf("Failed: compress\n"); free(src); free(dst); return 0; } /* Seekable random access */ zxc_seekable* s = zxc_seekable_open(dst, (size_t)csize); if (!s) { printf("Failed: open\n"); free(src); free(dst); return 0; } /* Range from block 0 */ uint8_t out[512]; int64_t r = zxc_seekable_decompress_range(s, out, sizeof(out), 0, 512); if (r != 512 || memcmp(out, src, 512) != 0) { printf("Failed: checksum range head mismatch\n"); zxc_seekable_free(s); free(src); free(dst); return 0; } /* Range spanning blocks 1-2 */ const uint64_t off = 64 * 1024 + 100; r = zxc_seekable_decompress_range(s, out, sizeof(out), off, 512); if (r != 512 || memcmp(out, src + off, 512) != 0) { printf("Failed: checksum range mid mismatch\n"); zxc_seekable_free(s); free(src); free(dst); return 0; } /* Full decompress with checksum verification */ uint8_t* full = malloc(SRC_SIZE); if (!full) { zxc_seekable_free(s); free(src); free(dst); return 0; } zxc_decompress_opts_t dopts = {.checksum_enabled = 1}; int64_t dsize = zxc_decompress(dst, (size_t)csize, full, SRC_SIZE, &dopts); if (dsize != (int64_t)SRC_SIZE || memcmp(src, full, SRC_SIZE) != 0) { printf("Failed: full decompress with checksum mismatch\n"); free(full); zxc_seekable_free(s); free(src); free(dst); return 0; } free(full); zxc_seekable_free(s); free(src); free(dst); printf("PASS\n\n"); return 1; } zxc-0.11.0/tests/test_seekable_mt.c000066400000000000000000000130631520102567100172170ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ #include "test_common.h" int test_seekable_mt_roundtrip() { printf("=== TEST: Seekable MT - Roundtrip (4 threads) ===\n"); const size_t SRC_SIZE = 512 * 1024; uint8_t* src = malloc(SRC_SIZE); if (!src) return 0; fill_seek_data(src, SRC_SIZE, 55); const size_t dst_cap = (size_t)zxc_compress_bound(SRC_SIZE) + 1024; uint8_t* dst = malloc(dst_cap); uint8_t* dec = malloc(SRC_SIZE); if (!dst || !dec) { free(src); free(dst); free(dec); return 0; } /* Compress with small blocks to create many blocks */ zxc_compress_opts_t opts = {.level = 3, .block_size = 64 * 1024, .seekable = 1}; const int64_t csize = zxc_compress(src, SRC_SIZE, dst, dst_cap, &opts); if (csize <= 0) { printf("Failed: compress\n"); free(src); free(dst); free(dec); return 0; } zxc_seekable* s = zxc_seekable_open(dst, (size_t)csize); if (!s) { printf("Failed: open\n"); free(src); free(dst); free(dec); return 0; } /* Full decompression with 4 threads */ int64_t r = zxc_seekable_decompress_range_mt(s, dec, SRC_SIZE, 0, SRC_SIZE, 4); if (r != (int64_t)SRC_SIZE || memcmp(src, dec, SRC_SIZE) != 0) { printf("Failed: MT decompress mismatch\n"); zxc_seekable_free(s); free(src); free(dst); free(dec); return 0; } zxc_seekable_free(s); free(src); free(dst); free(dec); printf("PASS\n\n"); return 1; } int test_seekable_mt_single_block() { printf("=== TEST: Seekable MT - Single Block Fallback ===\n"); const size_t SRC_SIZE = 32 * 1024; uint8_t* src = malloc(SRC_SIZE); if (!src) return 0; fill_seek_data(src, SRC_SIZE, 88); const size_t dst_cap = (size_t)zxc_compress_bound(SRC_SIZE) + 1024; uint8_t* dst = malloc(dst_cap); uint8_t* dec = malloc(SRC_SIZE); if (!dst || !dec) { free(src); free(dst); free(dec); return 0; } /* Compress with block_size >= SRC_SIZE => only 1 block */ zxc_compress_opts_t opts = {.level = 3, .block_size = 64 * 1024, .seekable = 1}; const int64_t csize = zxc_compress(src, SRC_SIZE, dst, dst_cap, &opts); if (csize <= 0) { printf("Failed: compress\n"); free(src); free(dst); free(dec); return 0; } zxc_seekable* s = zxc_seekable_open(dst, (size_t)csize); if (!s) { printf("Failed: open\n"); free(src); free(dst); free(dec); return 0; } /* Should fallback to ST since only 1 block */ int64_t r = zxc_seekable_decompress_range_mt(s, dec, SRC_SIZE, 0, SRC_SIZE, 4); if (r != (int64_t)SRC_SIZE || memcmp(src, dec, SRC_SIZE) != 0) { printf("Failed: single-block MT mismatch\n"); zxc_seekable_free(s); free(src); free(dst); free(dec); return 0; } zxc_seekable_free(s); free(src); free(dst); free(dec); printf("PASS\n\n"); return 1; } int test_seekable_mt_random_access() { printf("=== TEST: Seekable MT - Random Access ===\n"); const size_t SRC_SIZE = 512 * 1024; uint8_t* src = malloc(SRC_SIZE); if (!src) return 0; fill_seek_data(src, SRC_SIZE, 11); const size_t dst_cap = (size_t)zxc_compress_bound(SRC_SIZE) + 1024; uint8_t* dst = malloc(dst_cap); if (!dst) { free(src); return 0; } zxc_compress_opts_t opts = {.level = 3, .block_size = 64 * 1024, .seekable = 1}; const int64_t csize = zxc_compress(src, SRC_SIZE, dst, dst_cap, &opts); if (csize <= 0) { printf("Failed: compress\n"); free(src); free(dst); return 0; } zxc_seekable* s = zxc_seekable_open(dst, (size_t)csize); if (!s) { printf("Failed: open\n"); free(src); free(dst); return 0; } /* Mid-range spanning 3+ blocks with MT */ const uint64_t off = 100 * 1024; const size_t len = 200 * 1024; uint8_t* out = malloc(len); if (!out) { zxc_seekable_free(s); free(src); free(dst); return 0; } int64_t r = zxc_seekable_decompress_range_mt(s, out, len, off, len, 4); if (r != (int64_t)len || memcmp(src + off, out, len) != 0) { printf("Failed: MT random access mismatch\n"); free(out); zxc_seekable_free(s); free(src); free(dst); return 0; } free(out); zxc_seekable_free(s); free(src); free(dst); printf("PASS\n\n"); return 1; } int test_seekable_mt_full_file() { printf("=== TEST: Seekable MT - Full File (auto threads) ===\n"); const size_t SRC_SIZE = 1024 * 1024; /* 1 MB */ uint8_t* src = malloc(SRC_SIZE); if (!src) return 0; fill_seek_data(src, SRC_SIZE, 7); const size_t dst_cap = (size_t)zxc_compress_bound(SRC_SIZE) + 1024; uint8_t* dst = malloc(dst_cap); uint8_t* dec = malloc(SRC_SIZE); if (!dst || !dec) { free(src); free(dst); free(dec); return 0; } /* 16 blocks of 64K */ zxc_compress_opts_t opts = {.level = 3, .block_size = 64 * 1024, .seekable = 1}; const int64_t csize = zxc_compress(src, SRC_SIZE, dst, dst_cap, &opts); if (csize <= 0) { printf("Failed: compress\n"); free(src); free(dst); free(dec); return 0; } zxc_seekable* s = zxc_seekable_open(dst, (size_t)csize); if (!s) { printf("Failed: open\n"); free(src); free(dst); free(dec); return 0; } /* Decompress with auto-detect threads (n_threads=0) */ int64_t r = zxc_seekable_decompress_range_mt(s, dec, SRC_SIZE, 0, SRC_SIZE, 0); if (r != (int64_t)SRC_SIZE || memcmp(src, dec, SRC_SIZE) != 0) { printf("Failed: MT full file mismatch\n"); zxc_seekable_free(s); free(src); free(dst); free(dec); return 0; } zxc_seekable_free(s); free(src); free(dst); free(dec); printf("PASS\n\n"); return 1; } zxc-0.11.0/tests/test_stream_api.c000066400000000000000000001001721520102567100170660ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ #include "test_common.h" // Checks that the stream decompression can accept NULL output (Integrity Check Mode) int test_null_output_decompression() { printf("=== TEST: Unit - NULL Output Decompression (Integrity Check) ===\n"); size_t size = 64 * 1024; uint8_t* input = malloc(size); if (!input) return 0; gen_lz_data(input, size); FILE* f_in = tmpfile(); FILE* f_comp = tmpfile(); if (!f_in || !f_comp) { if (f_in) fclose(f_in); if (f_comp) fclose(f_comp); free(input); return 0; } fwrite(input, 1, size, f_in); fseek(f_in, 0, SEEK_SET); // Compress with checksum zxc_compress_opts_t _sco3 = {.n_threads = 1, .level = 3, .checksum_enabled = 1}; if (zxc_stream_compress(f_in, f_comp, &_sco3) < 0) { printf("Compression Failed!\n"); fclose(f_in); fclose(f_comp); free(input); return 0; } fseek(f_comp, 0, SEEK_SET); // Decompress with NULL output // This should return the decompressed size but write nothing zxc_decompress_opts_t _sdo4 = {.n_threads = 1, .checksum_enabled = 1}; int64_t d_sz = zxc_stream_decompress(f_comp, NULL, &_sdo4); if (d_sz != (int64_t)size) { printf("Failed: Expected size %zu, got %lld\n", size, (long long)d_sz); fclose(f_in); fclose(f_comp); free(input); return 0; } printf("PASS\n\n"); fclose(f_in); fclose(f_comp); free(input); return 1; } // Checks API robustness against invalid arguments int test_invalid_arguments() { printf("=== TEST: Unit - Invalid Arguments ===\n"); FILE* f = tmpfile(); if (!f) return 0; FILE* f_valid = tmpfile(); if (!f_valid) { fclose(f); return 0; } // Prepare a valid compressed stream for decompression tests zxc_compress_opts_t _sco5 = {.n_threads = 1, .level = 1, .checksum_enabled = 0}; zxc_stream_compress(f, f_valid, &_sco5); rewind(f_valid); // 1. Input NULL -> Must fail zxc_compress_opts_t _sco6 = {.n_threads = 1, .level = 5, .checksum_enabled = 0}; if (zxc_stream_compress(NULL, f, &_sco6) >= 0) { printf("Failed: Should return < 0 when Input is NULL\n"); fclose(f); return 0; } // 2. Output NULL -> Must SUCCEED (Benchmark / Dry-Run Mode) zxc_compress_opts_t _sco7 = {.n_threads = 1, .level = 5, .checksum_enabled = 0}; if (zxc_stream_compress(f, NULL, &_sco7) < 0) { printf("Failed: Should allow NULL Output (Benchmark mode support)\n"); fclose(f); return 0; } // 3. Decompression Input NULL -> Must fail zxc_decompress_opts_t _sdo8 = {.n_threads = 1, .checksum_enabled = 0}; if (zxc_stream_decompress(NULL, f, &_sdo8) >= 0) { printf("Failed: Decompress should return < 0 when Input is NULL\n"); fclose(f); return 0; } // 3b. Decompression Output NULL -> Must SUCCEED (Benchmark mode) zxc_decompress_opts_t _sdo9 = {.n_threads = 1, .checksum_enabled = 0}; if (zxc_stream_decompress(f_valid, NULL, &_sdo9) < 0) { printf("Failed: Decompress should allow NULL Output (Benchmark mode support)\n"); fclose(f_valid); return 0; } // 4. zxc_compress NULL checks zxc_compress_opts_t _co10 = {.level = 3, .checksum_enabled = 0}; if (zxc_compress(NULL, 100, (void*)1, 100, &_co10) >= 0) { printf("Failed: zxc_compress should return < 0 when src is NULL\n"); fclose(f); return 0; } zxc_compress_opts_t _co11 = {.level = 3, .checksum_enabled = 0}; if (zxc_compress((void*)1, 100, NULL, 100, &_co11) >= 0) { printf("Failed: zxc_compress should return < 0 when dst is NULL\n"); fclose(f); return 0; } // 5. zxc_decompress NULL checks zxc_decompress_opts_t _do12 = {.checksum_enabled = 0}; if (zxc_decompress(NULL, 100, (void*)1, 100, &_do12) >= 0) { printf("Failed: zxc_decompress should return < 0 when src is NULL\n"); fclose(f); return 0; } zxc_decompress_opts_t _do13 = {.checksum_enabled = 0}; if (zxc_decompress((void*)1, 100, NULL, 100, &_do13) >= 0) { printf("Failed: zxc_decompress should return < 0 when dst is NULL\n"); fclose(f); return 0; } // 6. zxc_compress_bound overflow check if (zxc_compress_bound(SIZE_MAX) != 0) { printf("Failed: zxc_compress_bound should return 0 on overflow\n"); fclose(f); return 0; } printf("PASS\n\n"); fclose(f); return 1; } // Checks behavior with truncated compressed input int test_truncated_input() { printf("=== TEST: Unit - Truncated Input (Stream) ===\n"); const size_t SRC_SIZE = 1024; uint8_t src[1024]; gen_lz_data(src, SRC_SIZE); size_t cap = (size_t)zxc_compress_bound(SRC_SIZE); uint8_t* compressed = malloc(cap); uint8_t* decomp_buf = malloc(SRC_SIZE); if (!compressed || !decomp_buf) { free(compressed); free(decomp_buf); return 0; } zxc_compress_opts_t _co14 = {.level = 3, .checksum_enabled = 1}; int64_t comp_sz = zxc_compress(src, SRC_SIZE, compressed, cap, &_co14); if (comp_sz <= 0) { printf("Prepare failed\n"); free(compressed); free(decomp_buf); return 0; } // Try decompressing with progressively cropped size // 1. Cut off the Footer (last ZXC_FILE_FOOTER_SIZE bytes) if (comp_sz > ZXC_FILE_FOOTER_SIZE) { zxc_decompress_opts_t _do15 = {.checksum_enabled = 1}; if (zxc_decompress(compressed, (size_t)(comp_sz - ZXC_FILE_FOOTER_SIZE), decomp_buf, SRC_SIZE, &_do15) >= 0) { printf("Failed: Should fail when footer is missing\n"); free(compressed); free(decomp_buf); return 0; } } // 2. Cut off half the file zxc_decompress_opts_t _do16 = {.checksum_enabled = 1}; if (zxc_decompress(compressed, (size_t)(comp_sz / 2), decomp_buf, SRC_SIZE, &_do16) >= 0) { printf("Failed: Should fail when stream is truncated by half\n"); free(compressed); free(decomp_buf); return 0; } // 3. Cut off just 1 byte zxc_decompress_opts_t _do17 = {.checksum_enabled = 1}; if (zxc_decompress(compressed, (size_t)(comp_sz - 1), decomp_buf, SRC_SIZE, &_do17) >= 0) { printf("Failed: Should fail when stream is truncated by 1 byte\n"); free(compressed); free(decomp_buf); return 0; } printf("PASS\n\n"); free(compressed); free(decomp_buf); return 1; } // Checks behavior if writing fails int test_io_failures() { printf("=== TEST: Unit - I/O Failures ===\n"); FILE* f_in = tmpfile(); if (!f_in) return 0; // Create a dummy file to simulate failure // Open it in "rb" (read-only) and pass it as "wb" output file. // fwrite should return 0 and trigger the error. const char* bad_filename = "zxc_test_readonly.tmp"; FILE* f_dummy = create_restricted_file(bad_filename); if (f_dummy) fclose(f_dummy); FILE* f_out = fopen(bad_filename, "rb"); if (!f_out) { perror("fopen readonly"); fclose(f_in); return 0; } // Write some data to input fputs("test data to compress", f_in); fseek(f_in, 0, SEEK_SET); // This should fail cleanly (return < 0) because writing to f_out is impossible zxc_compress_opts_t _sco18 = {.n_threads = 1, .level = 5, .checksum_enabled = 0}; if (zxc_stream_compress(f_in, f_out, &_sco18) >= 0) { printf("Failed: Should detect write error on read-only stream\n"); fclose(f_in); fclose(f_out); remove(bad_filename); return 0; } printf("PASS\n\n"); fclose(f_in); fclose(f_out); remove(bad_filename); return 1; } // Checks thread selector behavior int test_thread_params() { printf("=== TEST: Unit - Thread Parameters ===\n"); FILE* f_in = tmpfile(); FILE* f_out = tmpfile(); if (!f_in || !f_out) { if (f_in) fclose(f_in); if (f_out) fclose(f_out); return 0; } // Test with 0 (Auto) and negative value - must not crash zxc_compress_opts_t _sco19 = {.n_threads = 0, .level = 5, .checksum_enabled = 0}; zxc_stream_compress(f_in, f_out, &_sco19); fseek(f_in, 0, SEEK_SET); fseek(f_out, 0, SEEK_SET); zxc_compress_opts_t _sco20 = {.n_threads = -5, .level = 5, .checksum_enabled = 0}; zxc_stream_compress(f_in, f_out, &_sco20); printf("PASS (No crash observed)\n\n"); fclose(f_in); fclose(f_out); return 1; } // Multi-threaded round-trip test for TSan coverage int test_multithread_roundtrip() { printf("=== TEST: Multi-Thread Round-Trip (TSan Coverage) ===\n"); const size_t SIZE = 4 * 1024 * 1024; // 4MB to ensure multiple chunks const int ITERATIONS = 3; // Multiple runs increase race detection int result = 0; uint8_t* input = malloc(SIZE); uint8_t* output = malloc(SIZE); if (!input || !output) goto cleanup; gen_lz_data(input, SIZE); for (int iter = 0; iter < ITERATIONS; iter++) { FILE* f_in = tmpfile(); FILE* f_comp = tmpfile(); FILE* f_decomp = tmpfile(); if (!f_in || !f_comp || !f_decomp) { if (f_in) fclose(f_in); if (f_comp) fclose(f_comp); if (f_decomp) fclose(f_decomp); goto cleanup; } fwrite(input, 1, SIZE, f_in); fseek(f_in, 0, SEEK_SET); // Vary thread count: 2, 4, 8 int num_threads = 2 << iter; zxc_compress_opts_t _sco21 = {.n_threads = num_threads, .level = 3, .checksum_enabled = 1}; if (zxc_stream_compress(f_in, f_comp, &_sco21) < 0) { printf("Compression failed (threads=%d)!\n", num_threads); fclose(f_in); fclose(f_comp); fclose(f_decomp); goto cleanup; } fseek(f_comp, 0, SEEK_SET); zxc_decompress_opts_t _sdo22 = {.n_threads = num_threads, .checksum_enabled = 1}; if (zxc_stream_decompress(f_comp, f_decomp, &_sdo22) < 0) { printf("Decompression failed (threads=%d)!\n", num_threads); fclose(f_in); fclose(f_comp); fclose(f_decomp); goto cleanup; } long decomp_size = ftell(f_decomp); fseek(f_decomp, 0, SEEK_SET); if (decomp_size != (long)SIZE || fread(output, 1, SIZE, f_decomp) != SIZE || memcmp(input, output, SIZE) != 0) { printf("Verification failed (threads=%d)!\n", num_threads); fclose(f_in); fclose(f_comp); fclose(f_decomp); goto cleanup; } fclose(f_in); fclose(f_comp); fclose(f_decomp); printf(" Iteration %d: PASS (%d threads)\n", iter + 1, num_threads); } printf("PASS (3 iterations, 2/4/8 threads)\n\n"); result = 1; cleanup: free(input); free(output); return result; } int test_stream_get_decompressed_size_errors() { printf("=== TEST: Unit - zxc_stream_get_decompressed_size Error Codes ===\n"); // 1. NULL FILE* int64_t r = zxc_stream_get_decompressed_size(NULL); if (r != ZXC_ERROR_NULL_INPUT) { printf(" [FAIL] NULL FILE*: expected %d, got %lld\n", ZXC_ERROR_NULL_INPUT, (long long)r); return 0; } printf(" [PASS] NULL FILE* -> ZXC_ERROR_NULL_INPUT\n"); // 2. File too small (less than header + footer) { FILE* f = tmpfile(); if (!f) { printf(" [SKIP] tmpfile failed\n"); return 0; } fwrite("tiny", 1, 4, f); fseek(f, 0, SEEK_SET); r = zxc_stream_get_decompressed_size(f); if (r != ZXC_ERROR_SRC_TOO_SMALL) { printf(" [FAIL] file too small: expected %d, got %lld\n", ZXC_ERROR_SRC_TOO_SMALL, (long long)r); fclose(f); return 0; } fclose(f); } printf(" [PASS] file too small -> ZXC_ERROR_SRC_TOO_SMALL\n"); // 3. Bad magic word { FILE* f = tmpfile(); if (!f) { printf(" [SKIP] tmpfile failed\n"); return 0; } // Write enough bytes but with wrong magic uint8_t garbage[ZXC_FILE_HEADER_SIZE + ZXC_FILE_FOOTER_SIZE]; memset(garbage, 0, sizeof(garbage)); fwrite(garbage, 1, sizeof(garbage), f); fseek(f, 0, SEEK_SET); r = zxc_stream_get_decompressed_size(f); if (r != ZXC_ERROR_BAD_MAGIC) { printf(" [FAIL] bad magic: expected %d, got %lld\n", ZXC_ERROR_BAD_MAGIC, (long long)r); fclose(f); return 0; } fclose(f); } printf(" [PASS] bad magic -> ZXC_ERROR_BAD_MAGIC\n"); // 4. Valid file returns correct size { // Create a valid compressed file in memory const size_t src_sz = 512; uint8_t* src = malloc(src_sz); gen_lz_data(src, src_sz); const size_t cap = (size_t)zxc_compress_bound(src_sz); uint8_t* comp = malloc(cap); zxc_compress_opts_t _co48 = {.level = 3, .checksum_enabled = 0}; int64_t comp_sz = zxc_compress(src, src_sz, comp, cap, &_co48); if (comp_sz <= 0) { printf(" [SKIP] compress failed\n"); free(src); free(comp); return 0; } FILE* f = tmpfile(); fwrite(comp, 1, (size_t)comp_sz, f); fseek(f, 0, SEEK_SET); r = zxc_stream_get_decompressed_size(f); if (r != (int64_t)src_sz) { printf(" [FAIL] valid file: expected %zu, got %lld\n", src_sz, (long long)r); fclose(f); free(src); free(comp); return 0; } fclose(f); free(src); free(comp); } printf(" [PASS] valid file -> correct size\n"); printf("PASS\n\n"); return 1; } int test_stream_engine_errors() { printf("=== TEST: Unit - Stream Engine Error Codes ===\n"); // 1. zxc_stream_compress with NULL f_in { FILE* f_out = tmpfile(); zxc_compress_opts_t _sco49 = {.n_threads = 1, .level = 3, .checksum_enabled = 0}; int64_t r = zxc_stream_compress(NULL, f_out, &_sco49); if (r != ZXC_ERROR_NULL_INPUT) { printf(" [FAIL] compress NULL f_in: expected %d, got %lld\n", ZXC_ERROR_NULL_INPUT, (long long)r); if (f_out) fclose(f_out); return 0; } if (f_out) fclose(f_out); } printf(" [PASS] zxc_stream_compress NULL f_in -> ZXC_ERROR_NULL_INPUT\n"); // 2. zxc_stream_decompress with NULL f_in { FILE* f_out = tmpfile(); zxc_decompress_opts_t _sdo50 = {.n_threads = 1, .checksum_enabled = 0}; int64_t r = zxc_stream_decompress(NULL, f_out, &_sdo50); if (r != ZXC_ERROR_NULL_INPUT) { printf(" [FAIL] decompress NULL f_in: expected %d, got %lld\n", ZXC_ERROR_NULL_INPUT, (long long)r); if (f_out) fclose(f_out); return 0; } if (f_out) fclose(f_out); } printf(" [PASS] zxc_stream_decompress NULL f_in -> ZXC_ERROR_NULL_INPUT\n"); // 3. zxc_stream_decompress with bad header (invalid file) { FILE* f_in = tmpfile(); FILE* f_out = tmpfile(); if (!f_in || !f_out) { if (f_in) fclose(f_in); if (f_out) fclose(f_out); printf(" [SKIP] tmpfile failed\n"); return 0; } // Write garbage data (bad magic) uint8_t garbage[64]; memset(garbage, 0xAA, sizeof(garbage)); fwrite(garbage, 1, sizeof(garbage), f_in); fseek(f_in, 0, SEEK_SET); zxc_decompress_opts_t _sdo51 = {.n_threads = 1, .checksum_enabled = 0}; int64_t r = zxc_stream_decompress(f_in, f_out, &_sdo51); if (r != ZXC_ERROR_BAD_HEADER) { printf(" [FAIL] bad header: expected %d, got %lld\n", ZXC_ERROR_BAD_HEADER, (long long)r); fclose(f_in); fclose(f_out); return 0; } fclose(f_in); fclose(f_out); } printf(" [PASS] zxc_stream_decompress bad header -> ZXC_ERROR_BAD_HEADER\n"); // 4. Stream decompress with corrupted footer (stored size mismatch) { // First, create a valid compressed stream const size_t src_sz = 4096; uint8_t* src = malloc(src_sz); gen_lz_data(src, src_sz); FILE* f_comp_in = tmpfile(); FILE* f_comp_out = tmpfile(); fwrite(src, 1, src_sz, f_comp_in); fseek(f_comp_in, 0, SEEK_SET); zxc_compress_opts_t _sco52 = {.n_threads = 1, .level = 3, .checksum_enabled = 1}; int64_t comp_sz = zxc_stream_compress(f_comp_in, f_comp_out, &_sco52); fclose(f_comp_in); if (comp_sz <= 0) { printf(" [SKIP] stream compress failed\n"); fclose(f_comp_out); free(src); return 0; } // Read the compressed data, corrupt the footer source size, rewrite fseek(f_comp_out, 0, SEEK_END); const long comp_file_sz = ftell(f_comp_out); uint8_t* comp_data = malloc(comp_file_sz); fseek(f_comp_out, 0, SEEK_SET); if (fread(comp_data, 1, comp_file_sz, f_comp_out) != (size_t)comp_file_sz) { printf(" [FAIL] fread failed\n"); fclose(f_comp_out); free(comp_data); free(src); return 0; } fclose(f_comp_out); // Corrupt the stored source size in footer (last 12 bytes: [src_size(8)] + [hash(4)]) const size_t footer_off = comp_file_sz - ZXC_FILE_FOOTER_SIZE; comp_data[footer_off] ^= 0x01; // Flip a bit in stored source size FILE* f_corrupt = tmpfile(); FILE* f_dec_out = tmpfile(); fwrite(comp_data, 1, comp_file_sz, f_corrupt); fseek(f_corrupt, 0, SEEK_SET); free(comp_data); zxc_decompress_opts_t _sdo53 = {.n_threads = 1, .checksum_enabled = 1}; int64_t r = zxc_stream_decompress(f_corrupt, f_dec_out, &_sdo53); fclose(f_corrupt); fclose(f_dec_out); free(src); if (r >= 0) { printf(" [FAIL] corrupt footer size: expected < 0, got %lld\n", (long long)r); return 0; } } printf(" [PASS] zxc_stream_decompress corrupt footer -> negative\n"); // 5. Stream decompress with corrupted global checksum { const size_t src_sz = 4096; uint8_t* src = malloc(src_sz); gen_lz_data(src, src_sz); FILE* f_comp_in = tmpfile(); FILE* f_comp_out = tmpfile(); fwrite(src, 1, src_sz, f_comp_in); fseek(f_comp_in, 0, SEEK_SET); zxc_compress_opts_t _sco54 = {.n_threads = 1, .level = 3, .checksum_enabled = 1}; int64_t comp_sz = zxc_stream_compress(f_comp_in, f_comp_out, &_sco54); fclose(f_comp_in); if (comp_sz <= 0) { printf(" [SKIP] stream compress failed\n"); fclose(f_comp_out); free(src); return 0; } fseek(f_comp_out, 0, SEEK_END); const long comp_file_sz = ftell(f_comp_out); uint8_t* comp_data = malloc(comp_file_sz); fseek(f_comp_out, 0, SEEK_SET); if (fread(comp_data, 1, comp_file_sz, f_comp_out) != (size_t)comp_file_sz) { printf(" [FAIL] fread failed\n"); fclose(f_comp_out); free(comp_data); free(src); return 0; } fclose(f_comp_out); // Corrupt the global checksum (last 4 bytes) comp_data[comp_file_sz - 1] ^= 0xFF; FILE* f_corrupt = tmpfile(); FILE* f_dec_out = tmpfile(); fwrite(comp_data, 1, comp_file_sz, f_corrupt); fseek(f_corrupt, 0, SEEK_SET); free(comp_data); zxc_decompress_opts_t _sdo55 = {.n_threads = 1, .checksum_enabled = 1}; int64_t r = zxc_stream_decompress(f_corrupt, f_dec_out, &_sdo55); fclose(f_corrupt); fclose(f_dec_out); free(src); if (r >= 0) { printf(" [FAIL] corrupt global checksum: expected < 0, got %lld\n", (long long)r); return 0; } } printf(" [PASS] zxc_stream_decompress corrupt checksum -> negative\n"); // 6. Stream decompress truncated file (missing EOF + footer) { const size_t src_sz = 4096; uint8_t* src = malloc(src_sz); gen_lz_data(src, src_sz); FILE* f_comp_in = tmpfile(); FILE* f_comp_out = tmpfile(); fwrite(src, 1, src_sz, f_comp_in); fseek(f_comp_in, 0, SEEK_SET); zxc_compress_opts_t _sco56 = {.n_threads = 1, .level = 3, .checksum_enabled = 0}; int64_t comp_sz = zxc_stream_compress(f_comp_in, f_comp_out, &_sco56); fclose(f_comp_in); free(src); if (comp_sz <= 0) { printf(" [SKIP] stream compress failed\n"); fclose(f_comp_out); return 0; } fseek(f_comp_out, 0, SEEK_END); const long comp_file_sz = ftell(f_comp_out); // Truncate: remove the EOF block header + footer const long trunc_sz = comp_file_sz - (ZXC_BLOCK_HEADER_SIZE + ZXC_FILE_FOOTER_SIZE); uint8_t* comp_data = malloc(trunc_sz); fseek(f_comp_out, 0, SEEK_SET); if (fread(comp_data, 1, trunc_sz, f_comp_out) != (size_t)trunc_sz) { printf(" [FAIL] fread failed\n"); fclose(f_comp_out); free(comp_data); return 0; } fclose(f_comp_out); FILE* f_corrupt = tmpfile(); FILE* f_dec_out = tmpfile(); fwrite(comp_data, 1, trunc_sz, f_corrupt); fseek(f_corrupt, 0, SEEK_SET); free(comp_data); zxc_decompress_opts_t _sdo57 = {.n_threads = 1, .checksum_enabled = 0}; int64_t r = zxc_stream_decompress(f_corrupt, f_dec_out, &_sdo57); fclose(f_corrupt); fclose(f_dec_out); // Should fail: missing EOF/footer means io_error or bad read if (r >= 0) { printf(" [FAIL] truncated stream: expected < 0, got %lld\n", (long long)r); return 0; } } printf(" [PASS] zxc_stream_decompress truncated -> negative\n"); // 7. Stream decompress with mid-block body truncation { const size_t src_sz = 64 * 1024; uint8_t* src = malloc(src_sz); gen_lz_data(src, src_sz); FILE* f_comp_in = tmpfile(); FILE* f_comp_out = tmpfile(); fwrite(src, 1, src_sz, f_comp_in); fseek(f_comp_in, 0, SEEK_SET); zxc_compress_opts_t sco_mb = {.n_threads = 1, .level = 3, .checksum_enabled = 0}; int64_t comp_sz = zxc_stream_compress(f_comp_in, f_comp_out, &sco_mb); fclose(f_comp_in); free(src); if (comp_sz <= 0) { printf(" [SKIP] stream compress failed\n"); fclose(f_comp_out); return 0; } fseek(f_comp_out, 0, SEEK_END); const long comp_file_sz = ftell(f_comp_out); // Truncate mid-block: keep header + first block header + partial body const long trunc_sz = ZXC_FILE_HEADER_SIZE + ZXC_BLOCK_HEADER_SIZE + 16; if (trunc_sz < comp_file_sz) { uint8_t* comp_data = malloc(trunc_sz); fseek(f_comp_out, 0, SEEK_SET); if (fread(comp_data, 1, trunc_sz, f_comp_out) == (size_t)trunc_sz) { FILE* f_trunc = tmpfile(); FILE* f_dec_out = tmpfile(); fwrite(comp_data, 1, trunc_sz, f_trunc); fseek(f_trunc, 0, SEEK_SET); zxc_decompress_opts_t sdo_mb = {.n_threads = 1, .checksum_enabled = 0}; int64_t r = zxc_stream_decompress(f_trunc, f_dec_out, &sdo_mb); fclose(f_trunc); fclose(f_dec_out); if (r >= 0) { printf(" [FAIL] mid-block truncated: expected < 0, got %lld\n", (long long)r); free(comp_data); fclose(f_comp_out); return 0; } } free(comp_data); } fclose(f_comp_out); } printf(" [PASS] zxc_stream_decompress mid-block truncated -> negative\n"); // 8. Streaming fwrite error: compress real data, then decompress to a read-only file { const size_t src_sz = 64 * 1024; uint8_t* src = malloc(src_sz); gen_lz_data(src, src_sz); FILE* f_comp_in = tmpfile(); FILE* f_comp_out = tmpfile(); fwrite(src, 1, src_sz, f_comp_in); fseek(f_comp_in, 0, SEEK_SET); free(src); zxc_compress_opts_t sco_io = {.n_threads = 1, .level = 1, .checksum_enabled = 0}; int64_t comp_sz = zxc_stream_compress(f_comp_in, f_comp_out, &sco_io); fclose(f_comp_in); if (comp_sz <= 0) { printf(" [SKIP] compress failed\n"); fclose(f_comp_out); return 0; } fseek(f_comp_out, 0, SEEK_SET); // Open a read-only file as the output: fwrite will fail const char* ro_file = "zxc_test_stream_readonly.tmp"; FILE* f_ro = create_restricted_file(ro_file); if (f_ro) fclose(f_ro); FILE* f_bad_out = fopen(ro_file, "rb"); if (f_bad_out) { zxc_decompress_opts_t sdo_io = {.n_threads = 1, .checksum_enabled = 0}; int64_t r = zxc_stream_decompress(f_comp_out, f_bad_out, &sdo_io); fclose(f_bad_out); if (r >= 0) { printf(" [FAIL] fwrite error: expected < 0, got %lld\n", (long long)r); fclose(f_comp_out); remove(ro_file); return 0; } } fclose(f_comp_out); remove(ro_file); } printf(" [PASS] zxc_stream_decompress fwrite error -> negative\n"); // 9. Multi-threaded streaming I/O failure (writer fwrite error with multiple workers) { const size_t src_sz = 256 * 1024; uint8_t* src = malloc(src_sz); gen_lz_data(src, src_sz); FILE* f_comp_in = tmpfile(); FILE* f_comp_out = tmpfile(); fwrite(src, 1, src_sz, f_comp_in); fseek(f_comp_in, 0, SEEK_SET); free(src); zxc_compress_opts_t sco_mt = {.n_threads = 4, .level = 1, .checksum_enabled = 0}; int64_t comp_sz = zxc_stream_compress(f_comp_in, f_comp_out, &sco_mt); fclose(f_comp_in); if (comp_sz <= 0) { printf(" [SKIP] mt compress failed\n"); fclose(f_comp_out); return 0; } fseek(f_comp_out, 0, SEEK_SET); const char* ro_file2 = "zxc_test_stream_mt_readonly.tmp"; FILE* f_ro2 = create_restricted_file(ro_file2); if (f_ro2) fclose(f_ro2); FILE* f_bad_out2 = fopen(ro_file2, "rb"); if (f_bad_out2) { zxc_decompress_opts_t sdo_mt = {.n_threads = 4, .checksum_enabled = 0}; int64_t r = zxc_stream_decompress(f_comp_out, f_bad_out2, &sdo_mt); fclose(f_bad_out2); if (r >= 0) { printf(" [FAIL] mt fwrite error: expected < 0, got %lld\n", (long long)r); fclose(f_comp_out); remove(ro_file2); return 0; } } fclose(f_comp_out); remove(ro_file2); } printf(" [PASS] zxc_stream_decompress mt fwrite error -> negative\n"); printf("PASS\n\n"); return 1; } /* ======================================================================== */ /* Streaming round-trip suite */ /* */ /* Historical coverage: patterns x levels x checksum. Each case is its own */ /* named entry so CTest can schedule and report them individually. */ /* ======================================================================== */ /* Thin wrapper around test_round_trip: malloc, generate, run, free. */ #define RT_WRAPPER(fn_name, label, gen, size_expr, level_val, crc_val) \ int fn_name(void) { \ const size_t _sz = (size_expr); \ uint8_t* _buf = malloc(_sz > 0 ? _sz : 1); \ if (!_buf) return 0; \ gen(_buf, _sz); \ const int _ok = test_round_trip((label), _buf, _sz, (level_val), \ (crc_val)); \ free(_buf); \ return _ok; \ } #define RT_BUF (256 * 1024) #define RT_LARGE (15 * 1024 * 1024) /* Encoder path coverage */ RT_WRAPPER(test_roundtrip_raw_random, "RAW Block (Random Data)", gen_random_data, RT_BUF, 3, 0) RT_WRAPPER(test_roundtrip_ghi_text, "GHI Block (Text Pattern)", gen_lz_data, RT_BUF, 2, 0) RT_WRAPPER(test_roundtrip_glo_text, "GLO Block (Text Pattern)", gen_lz_data, RT_BUF, 4, 0) RT_WRAPPER(test_roundtrip_num_seq, "NUM Block (Integer Sequence)", gen_num_data, RT_BUF, 3, 0) RT_WRAPPER(test_roundtrip_num_zero, "NUM Block (Zero Deltas)", gen_num_data_zero, RT_BUF, 3, 0) RT_WRAPPER(test_roundtrip_num_small, "NUM Block (Small Deltas)", gen_num_data_small, RT_BUF, 3, 0) RT_WRAPPER(test_roundtrip_num_large, "NUM Block (Large Deltas)", gen_num_data_large, RT_BUF, 3, 0) /* Size edge cases */ RT_WRAPPER(test_roundtrip_small_50, "Small Input (50 bytes)", gen_random_data, 50, 3, 0) RT_WRAPPER(test_roundtrip_empty, "Empty Input (0 bytes)", gen_random_data, 0, 3, 0) RT_WRAPPER(test_roundtrip_1byte, "1-byte Input", gen_random_data, 1, 3, 0) RT_WRAPPER(test_roundtrip_1byte_crc, "1-byte Input (with checksum)", gen_random_data, 1, 3, 1) RT_WRAPPER(test_roundtrip_large_15mb_lz, "Large File (15MB Multi-Block)", gen_lz_data, RT_LARGE, 3, 1) RT_WRAPPER(test_roundtrip_large_15mb_num, "Large File NUM (15MB Multi-Block)", gen_num_data, RT_LARGE, 3, 1) /* Checksum coverage */ RT_WRAPPER(test_roundtrip_checksum_off, "Checksum Disabled", gen_lz_data, RT_BUF, 3, 0) RT_WRAPPER(test_roundtrip_checksum_on, "Checksum Enabled", gen_lz_data, RT_BUF, 31, 1) /* Per-level coverage */ RT_WRAPPER(test_roundtrip_level1, "Level 1", gen_lz_data, RT_BUF, 1, 1) RT_WRAPPER(test_roundtrip_level2, "Level 2", gen_lz_data, RT_BUF, 2, 1) RT_WRAPPER(test_roundtrip_level3, "Level 3", gen_lz_data, RT_BUF, 3, 1) RT_WRAPPER(test_roundtrip_level4, "Level 4", gen_lz_data, RT_BUF, 4, 1) RT_WRAPPER(test_roundtrip_level5, "Level 5", gen_lz_data, RT_BUF, 5, 1) RT_WRAPPER(test_roundtrip_level6, "Level 6 (Huffman literals)", gen_lz_data, RT_BUF, 6, 1) /* Binary data preservation */ RT_WRAPPER(test_roundtrip_binary, "Binary Data (0x00, 0x0A, 0x0D, 0xFF)", gen_binary_data, RT_BUF, 3, 0) RT_WRAPPER(test_roundtrip_binary_crc, "Binary Data with Checksum", gen_binary_data, RT_BUF, 3, 1) RT_WRAPPER(test_roundtrip_binary_small, "Small Binary Data (128 bytes)", gen_binary_data, 128, 3, 0) /* Repetitive pattern / offset encoding */ RT_WRAPPER(test_roundtrip_offset8_small, "8-bit Offsets (Small Pattern)", gen_small_offset_data, RT_BUF, 3, 1) RT_WRAPPER(test_roundtrip_offset8_lvl5, "8-bit Offsets (Level 5)", gen_small_offset_data, RT_BUF, 5, 1) RT_WRAPPER(test_roundtrip_offset16_large, "16-bit Offsets (Large Distance)", gen_large_offset_data, RT_BUF, 3, 1) RT_WRAPPER(test_roundtrip_offset16_lvl5, "16-bit Offsets (Level 5)", gen_large_offset_data, RT_BUF, 5, 1) /* Mixed offsets: two generators populate two halves of the buffer. */ int test_roundtrip_offset_mixed(void) { uint8_t* buf = malloc(RT_BUF); if (!buf) return 0; gen_small_offset_data(buf, RT_BUF / 2); gen_large_offset_data(buf + RT_BUF / 2, RT_BUF / 2); const int ok = test_round_trip("Mixed Offsets (Hybrid)", buf, RT_BUF, 3, 1); free(buf); return ok; } zxc-0.11.0/wrappers/000077500000000000000000000000001520102567100142375ustar00rootroot00000000000000zxc-0.11.0/wrappers/go/000077500000000000000000000000001520102567100146445ustar00rootroot00000000000000zxc-0.11.0/wrappers/go/README.md000066400000000000000000000042371520102567100161310ustar00rootroot00000000000000# ZXC Go Bindings High-performance Go bindings for the **ZXC** asymmetric compressor, optimised for **fast decompression**. Designed for *Write Once, Read Many* workloads like ML datasets, game assets, and caches. ## Features - **Blazing fast decompression** - ZXC is specifically optimised for read-heavy workloads. - **Buffer API** - compress/decompress `[]byte` slices in a single call. - **Streaming API** - multi-threaded file compression/decompression. - **Functional options** - clean, composable configuration (`WithLevel`, `WithChecksum`, `WithThreads`). - **Typed errors** - sentinel error values for every ZXC error code. ## Prerequisites Build the ZXC core library as a static library: ```bash cd /path/to/zxc cmake -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF \ -DZXC_BUILD_CLI=OFF -DZXC_BUILD_TESTS=OFF cmake --build build ``` ## Installation (from source) ```bash git clone https://github.com/hellobertrand/zxc.git cd zxc/wrappers/go CGO_ENABLED=1 CGO_LDFLAGS="-L../../build -lzxc -lpthread" go build ./... ``` ## Quick Start ```go package main import ( "fmt" "log" zxc "github.com/hellobertrand/zxc/wrappers/go" ) func main() { data := []byte("Hello, ZXC from Go!") // Compress compressed, err := zxc.Compress(data, zxc.WithLevel(zxc.LevelDefault)) if err != nil { log.Fatal(err) } fmt.Printf("Compressed %d => %d bytes\n", len(data), len(compressed)) // Decompress original, err := zxc.Decompress(compressed) if err != nil { log.Fatal(err) } fmt.Printf("Decompressed: %s\n", original) } ``` ## Streaming Files ```go // Multi-threaded file compression n, err := zxc.CompressFile("input.bin", "output.zxc", zxc.WithLevel(zxc.LevelCompact), zxc.WithThreads(4), zxc.WithChecksum(true), ) // Decompress n, err = zxc.DecompressFile("output.zxc", "restored.bin") ``` ## Testing ```bash cd zxc/wrappers/go CGO_ENABLED=1 CGO_LDFLAGS="-L../../build -lzxc -lpthread" go test -v -count=1 ./... ``` ## Benchmarks ```bash CGO_ENABLED=1 CGO_LDFLAGS="-L../../build -lzxc -lpthread" go test -bench=. -benchmem ./... ``` ## License BSD-3-Clause - see [LICENSE](../../LICENSE). zxc-0.11.0/wrappers/go/go.mod000066400000000000000000000000711520102567100157500ustar00rootroot00000000000000module github.com/hellobertrand/zxc/wrappers/go go 1.21 zxc-0.11.0/wrappers/go/zxc.go000066400000000000000000000175051520102567100160070ustar00rootroot00000000000000/* ZXC - High-performance lossless compression Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. SPDX-License-Identifier: BSD-3-Clause */ // Package zxc provides Go bindings to the ZXC high-performance lossless // compression library. // // ZXC is optimised for fast decompression speed: it is designed for // "Write Once, Read Many" workloads such as ML datasets, game assets, // firmware images and caches. // // # Quick Start // // compressed, err := zxc.Compress(data) // if err != nil { log.Fatal(err) } // // original, err := zxc.Decompress(compressed) // if err != nil { log.Fatal(err) } // // # Streaming (file-based) // // n, err := zxc.CompressFile("input.bin", "output.zxc") // n, err = zxc.DecompressFile("output.zxc", "restored.bin") package zxc /* #cgo CFLAGS: -I${SRCDIR}/../../include -DZXC_STATIC_DEFINE #cgo LDFLAGS: -lpthread #include #include #include "zxc.h" */ import "C" import ( "errors" "fmt" ) // ============================================================================ // Compression Levels // ============================================================================ // Level represents a ZXC compression level. type Level int const ( // LevelFastest provides the fastest compression, best for real-time // applications (level 1). LevelFastest Level = C.ZXC_LEVEL_FASTEST // LevelFast provides fast compression, good for real-time applications // (level 2). LevelFast Level = C.ZXC_LEVEL_FAST // LevelDefault is the recommended default: ratio > LZ4, decode speed > LZ4 // (level 3). LevelDefault Level = C.ZXC_LEVEL_DEFAULT // LevelBalanced provides good ratio with good decode speed (level 4). LevelBalanced Level = C.ZXC_LEVEL_BALANCED // LevelCompact provides high density: storage, firmware and assets // (level 5). LevelCompact Level = C.ZXC_LEVEL_COMPACT // LevelDensity provides maximum density: Huffman-coded literals on top // of LevelCompact plus a price-based optimal LZ77 parser. Slowest // compression, best ratio (level 6). LevelDensity Level = C.ZXC_LEVEL_DENSITY ) // AllLevels returns all available compression levels from fastest to most // compact. func AllLevels() []Level { return []Level{LevelFastest, LevelFast, LevelDefault, LevelBalanced, LevelCompact, LevelDensity} } // ============================================================================ // Version // ============================================================================ const ( // VersionMajor is the major version of the underlying C library. VersionMajor = C.ZXC_VERSION_MAJOR // VersionMinor is the minor version of the underlying C library. VersionMinor = C.ZXC_VERSION_MINOR // VersionPatch is the patch version of the underlying C library. VersionPatch = C.ZXC_VERSION_PATCH ) // Version returns the library version as (major, minor, patch). func Version() (int, int, int) { return int(VersionMajor), int(VersionMinor), int(VersionPatch) } // VersionString returns the library version as a "major.minor.patch" string, // computed from the compile-time constants in the C header. // // For the version reported by the dynamically-linked native library (useful // for detecting ABI mismatch), call [LibraryVersion]. func VersionString() string { return fmt.Sprintf("%d.%d.%d", VersionMajor, VersionMinor, VersionPatch) } // LibraryVersion returns the version string reported by the linked native // libzxc (e.g. "0.11.0"). func LibraryVersion() string { return C.GoString(C.zxc_version_string()) } // MinLevel returns the minimum supported compression level (currently 1). func MinLevel() int { return int(C.zxc_min_level()) } // MaxLevel returns the maximum supported compression level (currently 6). func MaxLevel() int { return int(C.zxc_max_level()) } // DefaultLevel returns the default compression level (currently 3). func DefaultLevel() int { return int(C.zxc_default_level()) } // ============================================================================ // Error Handling // ============================================================================ // Error represents a ZXC library error. type Error struct { Code int Name string } func (e *Error) Error() string { return fmt.Sprintf("zxc: %s (code %d)", e.Name, e.Code) } // Sentinel errors for each ZXC error code. var ( ErrMemory = &Error{Code: int(C.ZXC_ERROR_MEMORY), Name: "memory allocation failed"} ErrDstTooSmall = &Error{Code: int(C.ZXC_ERROR_DST_TOO_SMALL), Name: "destination buffer too small"} ErrSrcTooSmall = &Error{Code: int(C.ZXC_ERROR_SRC_TOO_SMALL), Name: "source buffer too small"} ErrBadMagic = &Error{Code: int(C.ZXC_ERROR_BAD_MAGIC), Name: "invalid magic word"} ErrBadVersion = &Error{Code: int(C.ZXC_ERROR_BAD_VERSION), Name: "unsupported format version"} ErrBadHeader = &Error{Code: int(C.ZXC_ERROR_BAD_HEADER), Name: "corrupted header"} ErrBadChecksum = &Error{Code: int(C.ZXC_ERROR_BAD_CHECKSUM), Name: "checksum verification failed"} ErrCorruptData = &Error{Code: int(C.ZXC_ERROR_CORRUPT_DATA), Name: "corrupted compressed data"} ErrBadOffset = &Error{Code: int(C.ZXC_ERROR_BAD_OFFSET), Name: "invalid match offset"} ErrOverflow = &Error{Code: int(C.ZXC_ERROR_OVERFLOW), Name: "buffer overflow detected"} ErrIO = &Error{Code: int(C.ZXC_ERROR_IO), Name: "I/O error"} ErrNullInput = &Error{Code: int(C.ZXC_ERROR_NULL_INPUT), Name: "null input pointer"} ErrBadBlockType = &Error{Code: int(C.ZXC_ERROR_BAD_BLOCK_TYPE), Name: "unknown block type"} ErrBadBlockSize = &Error{Code: int(C.ZXC_ERROR_BAD_BLOCK_SIZE), Name: "invalid block size"} ErrInvalidData = errors.New("zxc: invalid compressed data") ) // errorFromCode converts a negative C error code to a Go error. func errorFromCode(code C.int64_t) error { switch int(code) { case int(C.ZXC_ERROR_MEMORY): return ErrMemory case int(C.ZXC_ERROR_DST_TOO_SMALL): return ErrDstTooSmall case int(C.ZXC_ERROR_SRC_TOO_SMALL): return ErrSrcTooSmall case int(C.ZXC_ERROR_BAD_MAGIC): return ErrBadMagic case int(C.ZXC_ERROR_BAD_VERSION): return ErrBadVersion case int(C.ZXC_ERROR_BAD_HEADER): return ErrBadHeader case int(C.ZXC_ERROR_BAD_CHECKSUM): return ErrBadChecksum case int(C.ZXC_ERROR_CORRUPT_DATA): return ErrCorruptData case int(C.ZXC_ERROR_BAD_OFFSET): return ErrBadOffset case int(C.ZXC_ERROR_OVERFLOW): return ErrOverflow case int(C.ZXC_ERROR_IO): return ErrIO case int(C.ZXC_ERROR_NULL_INPUT): return ErrNullInput case int(C.ZXC_ERROR_BAD_BLOCK_TYPE): return ErrBadBlockType case int(C.ZXC_ERROR_BAD_BLOCK_SIZE): return ErrBadBlockSize default: return fmt.Errorf("zxc: unknown error (code %d)", int(code)) } } // ============================================================================ // Options (functional options pattern) // ============================================================================ // options holds all configurable parameters. type options struct { level Level checksum bool seekable bool threads int } func defaultOptions() options { return options{ level: LevelDefault, checksum: false, threads: 0, // auto-detect } } // Option configures compression or decompression. type Option func(*options) // WithLevel sets the compression level. func WithLevel(l Level) Option { return func(o *options) { o.level = l } } // WithChecksum enables checksum computation and verification. func WithChecksum(enabled bool) Option { return func(o *options) { o.checksum = enabled } } // WithThreads sets the number of worker threads for streaming operations. // A value of 0 means auto-detect the CPU core count. func WithThreads(n int) Option { return func(o *options) { o.threads = n } } // WithSeekable enables seek table generation for random-access decompression. func WithSeekable(enabled bool) Option { return func(o *options) { o.seekable = enabled } } func applyOptions(opts []Option) options { o := defaultOptions() for _, fn := range opts { fn(&o) } return o } zxc-0.11.0/wrappers/go/zxc_block.go000066400000000000000000000136051520102567100171560ustar00rootroot00000000000000/* ZXC - High-performance lossless compression Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. SPDX-License-Identifier: BSD-3-Clause */ package zxc /* #include "zxc.h" */ import "C" import "unsafe" // ============================================================================ // Block API (single block, no file framing) // ============================================================================ // CompressBlockBound returns the maximum compressed size for a single block // of inputSize bytes (no file framing, no EOF, no footer). func CompressBlockBound(inputSize int) uint64 { if inputSize < 0 { return 0 } return uint64(C.zxc_compress_block_bound(C.size_t(inputSize))) } // DecompressBlockBound returns the minimum destination buffer size required // by [Cctx.DecompressBlock] for a block of uncompressedSize bytes. // // The fast decoder uses speculative wild-copy writes and needs a small tail // pad beyond the declared uncompressed size. Callers that cannot oversize // their destination buffer should use [Dctx.DecompressBlockSafe] instead. func DecompressBlockBound(uncompressedSize int) uint64 { if uncompressedSize < 0 { return 0 } return uint64(C.zxc_decompress_block_bound(C.size_t(uncompressedSize))) } // EstimateCctxSize returns an accurate estimate of the memory a compression // context reserves when compressing a single block of srcSize bytes at the // given compression level via [Cctx.CompressBlock]. // // The estimate covers all per-chunk working buffers (chain table, literals, // sequence/token/offset/extras buffers) plus the fixed hash tables and // cache-line alignment padding. At level 6+ it also accounts for the // optimal-parser scratch. It scales roughly linearly with srcSize and is // intended for integrators that need an accurate memory budget. // // Returns 0 if srcSize is 0 or negative. func EstimateCctxSize(srcSize, level int) uint64 { if srcSize <= 0 { return 0 } return uint64(C.zxc_estimate_cctx_size(C.size_t(srcSize), C.int(level))) } // Cctx is a reusable compression context for the Block API. // // It wraps an opaque C handle that is freed by [Cctx.Close]. Create with // [NewCctx] and always defer a call to Close to release the native memory. type Cctx struct { ptr *C.zxc_cctx } // NewCctx creates a new compression context. // // Options [WithLevel], [WithChecksum] and [WithBlockSize] are supported at // creation time to pre-allocate internal buffers; otherwise allocation is // deferred to the first call to [Cctx.CompressBlock]. func NewCctx(opts ...Option) (*Cctx, error) { o := applyOptions(opts) var copts C.zxc_compress_opts_t copts.level = C.int(o.level) if o.checksum { copts.checksum_enabled = 1 } // Block size is optional; 0 lets the library pick the default. ptr := C.zxc_create_cctx(&copts) if ptr == nil { return nil, ErrMemory } return &Cctx{ptr: ptr}, nil } // Close releases the native resources held by the context. Safe to call // multiple times. func (c *Cctx) Close() error { if c == nil || c.ptr == nil { return nil } C.zxc_free_cctx(c.ptr) c.ptr = nil return nil } // CompressBlock compresses a single block using the context. Output format // is [block_header(8B) + payload (+ optional checksum 4B)]. Use // [CompressBlockBound] to size dst. func (c *Cctx) CompressBlock(src, dst []byte, opts ...Option) (int, error) { if c == nil || c.ptr == nil { return 0, ErrNullInput } if len(src) == 0 { return 0, ErrSrcTooSmall } if len(dst) == 0 { return 0, ErrDstTooSmall } o := applyOptions(opts) var copts C.zxc_compress_opts_t copts.level = C.int(o.level) if o.checksum { copts.checksum_enabled = 1 } n := C.zxc_compress_block( c.ptr, unsafe.Pointer(&src[0]), C.size_t(len(src)), unsafe.Pointer(&dst[0]), C.size_t(len(dst)), &copts, ) if n < 0 { return 0, errorFromCode(n) } return int(n), nil } // Dctx is a reusable decompression context for the Block API. type Dctx struct { ptr *C.zxc_dctx } // NewDctx creates a new decompression context. func NewDctx() (*Dctx, error) { ptr := C.zxc_create_dctx() if ptr == nil { return nil, ErrMemory } return &Dctx{ptr: ptr}, nil } // Close releases the native resources held by the context. func (d *Dctx) Close() error { if d == nil || d.ptr == nil { return nil } C.zxc_free_dctx(d.ptr) d.ptr = nil return nil } // DecompressBlock decompresses a single block produced by // [Cctx.CompressBlock]. // // dst should be at least [DecompressBlockBound](uncompressedSize) to enable // the fast decode path. For a strictly-sized destination buffer use // [Dctx.DecompressBlockSafe] instead. func (d *Dctx) DecompressBlock(src, dst []byte, opts ...Option) (int, error) { if d == nil || d.ptr == nil { return 0, ErrNullInput } if len(src) == 0 { return 0, ErrSrcTooSmall } if len(dst) == 0 { return 0, ErrDstTooSmall } o := applyOptions(opts) var dopts C.zxc_decompress_opts_t if o.checksum { dopts.checksum_enabled = 1 } n := C.zxc_decompress_block( d.ptr, unsafe.Pointer(&src[0]), C.size_t(len(src)), unsafe.Pointer(&dst[0]), C.size_t(len(dst)), &dopts, ) if n < 0 { return 0, errorFromCode(n) } return int(n), nil } // DecompressBlockSafe is a strict-sized variant of [Dctx.DecompressBlock]: // it accepts a destination buffer sized exactly to the uncompressed length, // with no tail-pad required. Slightly slower than the fast path; output is // bit-identical. func (d *Dctx) DecompressBlockSafe(src, dst []byte, opts ...Option) (int, error) { if d == nil || d.ptr == nil { return 0, ErrNullInput } if len(src) == 0 { return 0, ErrSrcTooSmall } if len(dst) == 0 { return 0, ErrDstTooSmall } o := applyOptions(opts) var dopts C.zxc_decompress_opts_t if o.checksum { dopts.checksum_enabled = 1 } n := C.zxc_decompress_block_safe( d.ptr, unsafe.Pointer(&src[0]), C.size_t(len(src)), unsafe.Pointer(&dst[0]), C.size_t(len(dst)), &dopts, ) if n < 0 { return 0, errorFromCode(n) } return int(n), nil } zxc-0.11.0/wrappers/go/zxc_buffer.go000066400000000000000000000077051520102567100173410ustar00rootroot00000000000000/* ZXC - High-performance lossless compression Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. SPDX-License-Identifier: BSD-3-Clause */ package zxc /* #include "zxc.h" */ import "C" import "unsafe" // ============================================================================ // Buffer API // ============================================================================ // CompressBound returns the maximum compressed size for an input of the given // size. Use this to pre-allocate output buffers. func CompressBound(inputSize int) uint64 { return uint64(C.zxc_compress_bound(C.size_t(inputSize))) } // Compress compresses data using the ZXC algorithm. // // Options: [WithLevel], [WithChecksum]. // // out, err := zxc.Compress(data, zxc.WithLevel(zxc.LevelCompact)) func Compress(data []byte, opts ...Option) ([]byte, error) { if len(data) == 0 { return nil, ErrSrcTooSmall } o := applyOptions(opts) bound := CompressBound(len(data)) dst := make([]byte, bound) var copts C.zxc_compress_opts_t copts.level = C.int(o.level) if o.checksum { copts.checksum_enabled = 1 } written := C.zxc_compress( unsafe.Pointer(&data[0]), C.size_t(len(data)), unsafe.Pointer(&dst[0]), C.size_t(bound), &copts, ) if written < 0 { return nil, errorFromCode(written) } if written == 0 { return nil, ErrInvalidData } return dst[:int(written)], nil } // CompressTo compresses data into a pre-allocated output buffer. // Returns the number of bytes written. func CompressTo(data []byte, output []byte, opts ...Option) (int, error) { if len(data) == 0 { return 0, ErrSrcTooSmall } if len(output) == 0 { return 0, ErrDstTooSmall } o := applyOptions(opts) var copts C.zxc_compress_opts_t copts.level = C.int(o.level) if o.checksum { copts.checksum_enabled = 1 } written := C.zxc_compress( unsafe.Pointer(&data[0]), C.size_t(len(data)), unsafe.Pointer(&output[0]), C.size_t(len(output)), &copts, ) if written < 0 { return 0, errorFromCode(written) } if written == 0 { return 0, ErrInvalidData } return int(written), nil } // DecompressedSize returns the original uncompressed size stored in the // compressed data footer. Returns 0, ErrInvalidData if the data is too small // or invalid. func DecompressedSize(data []byte) (uint64, error) { if len(data) == 0 { return 0, ErrInvalidData } size := C.zxc_get_decompressed_size( unsafe.Pointer(&data[0]), C.size_t(len(data)), ) if size == 0 { return 0, ErrInvalidData } return uint64(size), nil } // Decompress decompresses ZXC-compressed data. // // The output size is read from the compressed data footer. For pre-allocated // buffers, use [DecompressTo]. // // Options: [WithChecksum]. func Decompress(data []byte, opts ...Option) ([]byte, error) { if len(data) == 0 { return nil, ErrInvalidData } size, err := DecompressedSize(data) if err != nil { return nil, err } if size == 0 { return []byte{}, nil } o := applyOptions(opts) dst := make([]byte, size) var dopts C.zxc_decompress_opts_t if o.checksum { dopts.checksum_enabled = 1 } written := C.zxc_decompress( unsafe.Pointer(&data[0]), C.size_t(len(data)), unsafe.Pointer(&dst[0]), C.size_t(size), &dopts, ) if written < 0 { return nil, errorFromCode(written) } if uint64(written) != size { return nil, ErrInvalidData } return dst[:int(written)], nil } // DecompressTo decompresses data into a pre-allocated output buffer. // Returns the number of bytes written. func DecompressTo(data []byte, output []byte, opts ...Option) (int, error) { if len(data) == 0 { return 0, ErrInvalidData } if len(output) == 0 { return 0, ErrDstTooSmall } o := applyOptions(opts) var dopts C.zxc_decompress_opts_t if o.checksum { dopts.checksum_enabled = 1 } written := C.zxc_decompress( unsafe.Pointer(&data[0]), C.size_t(len(data)), unsafe.Pointer(&output[0]), C.size_t(len(output)), &dopts, ) if written < 0 { return 0, errorFromCode(written) } return int(written), nil } zxc-0.11.0/wrappers/go/zxc_dup_unix.go000066400000000000000000000025101520102567100177100ustar00rootroot00000000000000//go:build unix package zxc /* #include #include */ import "C" import ( "fmt" "os" "runtime" ) // C mode strings - allocated once, never freed (intentional; they live for the // process lifetime and avoid per-call C.CString/C.free overhead). var ( cModeRead = C.CString("rb") cModeWrite = C.CString("wb") ) // dupFileRead duplicates a Go *os.File's fd and wraps it in a C FILE* for // reading. The returned FILE* owns its own fd and must be closed with // C.fclose(). func dupFileRead(f *os.File) (*C.FILE, error) { fd := C.int(f.Fd()) dupFd := C.dup(fd) runtime.KeepAlive(f) if dupFd < 0 { return nil, fmt.Errorf("zxc: dup failed for read fd") } cFile := C.fdopen(dupFd, cModeRead) if cFile == nil { C.close(dupFd) return nil, fmt.Errorf("zxc: fdopen failed for read fd") } return cFile, nil } // dupFileWrite duplicates a Go *os.File's fd and wraps it in a C FILE* for // writing. The returned FILE* owns its own fd and must be closed with // C.fclose(). func dupFileWrite(f *os.File) (*C.FILE, error) { fd := C.int(f.Fd()) dupFd := C.dup(fd) runtime.KeepAlive(f) if dupFd < 0 { return nil, fmt.Errorf("zxc: dup failed for write fd") } cFile := C.fdopen(dupFd, cModeWrite) if cFile == nil { C.close(dupFd) return nil, fmt.Errorf("zxc: fdopen failed for write fd") } return cFile, nil } zxc-0.11.0/wrappers/go/zxc_dup_windows.go000066400000000000000000000036641520102567100204320ustar00rootroot00000000000000//go:build windows package zxc /* #include #include #include */ import "C" import ( "fmt" "os" "runtime" ) // C mode strings - allocated once, never freed (intentional; they live for the // process lifetime and avoid per-call C.CString/C.free overhead). var ( cModeRead = C.CString("rb") cModeWrite = C.CString("wb") ) // dupFileRead converts a Go *os.File to a C FILE* for reading on Windows. // // os.File.Fd() returns a Windows HANDLE (not a CRT fd), so we must use // _open_osfhandle() to wrap it as a CRT fd, then _dup() to own it // independently, since _open_osfhandle() transfers ownership of the HANDLE. func dupFileRead(f *os.File) (*C.FILE, error) { handle := C.intptr_t(f.Fd()) runtime.KeepAlive(f) // Wrap the HANDLE as a read-only CRT fd. crtFd := C._open_osfhandle(handle, C._O_RDONLY) if crtFd < 0 { return nil, fmt.Errorf("zxc: _open_osfhandle failed for read fd") } // Dup so we own this fd independently from Go's *os.File. // Do NOT close crtFd: _open_osfhandle transfers HANDLE ownership to the // CRT; closing it would also close the HANDLE still owned by Go. dupFd := C._dup(crtFd) if dupFd < 0 { return nil, fmt.Errorf("zxc: _dup failed for read fd") } cFile := C._fdopen(dupFd, cModeRead) if cFile == nil { C._close(dupFd) return nil, fmt.Errorf("zxc: _fdopen failed for read fd") } return cFile, nil } // dupFileWrite converts a Go *os.File to a C FILE* for writing on Windows. func dupFileWrite(f *os.File) (*C.FILE, error) { handle := C.intptr_t(f.Fd()) runtime.KeepAlive(f) crtFd := C._open_osfhandle(handle, C._O_WRONLY) if crtFd < 0 { return nil, fmt.Errorf("zxc: _open_osfhandle failed for write fd") } dupFd := C._dup(crtFd) if dupFd < 0 { return nil, fmt.Errorf("zxc: _dup failed for write fd") } cFile := C._fdopen(dupFd, cModeWrite) if cFile == nil { C._close(dupFd) return nil, fmt.Errorf("zxc: _fdopen failed for write fd") } return cFile, nil } zxc-0.11.0/wrappers/go/zxc_file.go000066400000000000000000000063741520102567100170100ustar00rootroot00000000000000/* ZXC - High-performance lossless compression Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. SPDX-License-Identifier: BSD-3-Clause */ package zxc /* #include #include "zxc.h" */ import "C" import ( "fmt" "os" ) // ============================================================================ // Streaming API (file-based) // ============================================================================ // CompressFile compresses src to dst using multi-threaded streaming. // // This is the recommended method for large files. It uses an asynchronous // pipeline with separate reader, worker, and writer threads. // // Options: [WithLevel], [WithChecksum], [WithThreads]. // // n, err := zxc.CompressFile("input.bin", "output.zxc", // zxc.WithLevel(zxc.LevelCompact), zxc.WithThreads(4)) func CompressFile(input, output string, opts ...Option) (int64, error) { o := applyOptions(opts) fIn, err := os.Open(input) if err != nil { return 0, fmt.Errorf("zxc: open input: %w", err) } defer fIn.Close() fOut, err := os.Create(output) if err != nil { return 0, fmt.Errorf("zxc: create output: %w", err) } defer fOut.Close() // Duplicate file descriptors so the C FILE* owns its own fd. cIn, err := dupFileRead(fIn) if err != nil { return 0, err } defer C.fclose(cIn) cOut, err := dupFileWrite(fOut) if err != nil { return 0, err } defer C.fclose(cOut) var copts C.zxc_compress_opts_t copts.n_threads = C.int(o.threads) copts.level = C.int(o.level) if o.checksum { copts.checksum_enabled = 1 } if o.seekable { copts.seekable = 1 } result := C.zxc_stream_compress(cIn, cOut, &copts) if result < 0 { return 0, errorFromCode(result) } return int64(result), nil } // FileDecompressedSize reads the original uncompressed size from the footer // of a ZXC compressed file without performing decompression. The file is // opened in "rb" mode; the underlying file position is restored internally. func FileDecompressedSize(path string) (int64, error) { f, err := os.Open(path) if err != nil { return 0, fmt.Errorf("zxc: open input: %w", err) } defer f.Close() cf, err := dupFileRead(f) if err != nil { return 0, err } defer C.fclose(cf) result := C.zxc_stream_get_decompressed_size(cf) if result < 0 { return 0, errorFromCode(result) } return int64(result), nil } // DecompressFile decompresses src to dst using multi-threaded streaming. // // Options: [WithChecksum], [WithThreads]. // // n, err := zxc.DecompressFile("compressed.zxc", "output.bin") func DecompressFile(input, output string, opts ...Option) (int64, error) { o := applyOptions(opts) fIn, err := os.Open(input) if err != nil { return 0, fmt.Errorf("zxc: open input: %w", err) } defer fIn.Close() fOut, err := os.Create(output) if err != nil { return 0, fmt.Errorf("zxc: create output: %w", err) } defer fOut.Close() cIn, err := dupFileRead(fIn) if err != nil { return 0, err } defer C.fclose(cIn) cOut, err := dupFileWrite(fOut) if err != nil { return 0, err } defer C.fclose(cOut) var dopts C.zxc_decompress_opts_t dopts.n_threads = C.int(o.threads) if o.checksum { dopts.checksum_enabled = 1 } result := C.zxc_stream_decompress(cIn, cOut, &dopts) if result < 0 { return 0, errorFromCode(result) } return int64(result), nil } zxc-0.11.0/wrappers/go/zxc_io.go000066400000000000000000000146401520102567100164730ustar00rootroot00000000000000/* ZXC - High-performance lossless compression Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. SPDX-License-Identifier: BSD-3-Clause */ package zxc import ( "io" ) // ============================================================================ // io.Reader / io.Writer adapters over the push streaming API // ============================================================================ // // These wrappers turn a [CStream] / [DStream] pair into the standard Go // streaming interfaces (io.Reader, io.Writer, io.Closer) so ZXC can be plugged // into pipelines that expect them — notably OCI-style content stores, tar // pipelines, HTTP transports, etc. // Magic word identifying a ZXC file frame: little-endian 0x9CB02EF5. var zxcMagicLE = [4]byte{0xF5, 0x2E, 0xB0, 0x9C} // DetectZxc reports whether data starts with the ZXC file magic word. // // Useful for content-type sniffing in containers / object stores that need to // decide which decoder to dispatch (e.g. OCI media-type negotiation). The // check is cheap and side-effect free; it does not validate the rest of the // header or the footer. func DetectZxc(data []byte) bool { if len(data) < 4 { return false } return data[0] == zxcMagicLE[0] && data[1] == zxcMagicLE[1] && data[2] == zxcMagicLE[2] && data[3] == zxcMagicLE[3] } // ---------------------------------------------------------------------------- // Writer // ---------------------------------------------------------------------------- // Writer is an [io.WriteCloser] that compresses bytes written to it and // forwards the resulting ZXC frame to an underlying writer. // // The caller MUST call [Writer.Close] to flush the residual block, EOF marker // and footer; closing the underlying writer is the caller's responsibility. // // Writer is not safe for concurrent use. type Writer struct { dst io.Writer cs *CStream outBuf []byte err error } // NewWriter returns a [Writer] that compresses into w using a single-threaded // push pipeline. // // Honoured options: [WithLevel], [WithChecksum]. [WithThreads] and // [WithSeekable] are ignored (push API is single-threaded). func NewWriter(w io.Writer, opts ...Option) (*Writer, error) { cs, err := NewCStream(opts...) if err != nil { return nil, err } return &Writer{ dst: w, cs: cs, outBuf: make([]byte, cs.OutSize()), }, nil } // Write compresses p and forwards the produced bytes to the underlying writer. // Returns the number of bytes consumed from p (always len(p) on success). func (w *Writer) Write(p []byte) (int, error) { if w.err != nil { return 0, w.err } if w.cs == nil { return 0, ErrNullInput } total := 0 for len(p) > 0 { consumed, produced, pending, err := w.cs.Compress(w.outBuf, p) if err != nil { w.err = err return total, err } if produced > 0 { if _, werr := w.dst.Write(w.outBuf[:produced]); werr != nil { w.err = werr return total, werr } } total += consumed p = p[consumed:] // If the encoder made no input progress, drain pending output and // retry — this guarantees forward progress even when the caller // supplies a smaller-than-recommended output buffer. if consumed == 0 && produced == 0 && pending == 0 { break } } return total, nil } // Close flushes any buffered data, writes the EOF block + footer, and releases // the underlying CStream. The wrapped io.Writer is NOT closed. // // Safe to call multiple times; subsequent calls are no-ops. func (w *Writer) Close() error { if w.cs == nil { return nil } defer func() { _ = w.cs.Close() w.cs = nil }() if w.err != nil { return w.err } for { produced, pending, err := w.cs.End(w.outBuf) if err != nil { w.err = err return err } if produced > 0 { if _, werr := w.dst.Write(w.outBuf[:produced]); werr != nil { w.err = werr return werr } } if pending == 0 { return nil } } } // ---------------------------------------------------------------------------- // Reader // ---------------------------------------------------------------------------- // Reader is an [io.ReadCloser] that decompresses a ZXC frame read from an // underlying reader. // // Reader is not safe for concurrent use. type Reader struct { src io.Reader ds *DStream inBuf []byte inPos int inLen int err error eof bool // src returned io.EOF } // NewReader returns a [Reader] that decompresses from r. // // Honoured options: [WithChecksum]. [WithThreads] is ignored. func NewReader(r io.Reader, opts ...Option) (*Reader, error) { ds, err := NewDStream(opts...) if err != nil { return nil, err } return &Reader{ src: r, ds: ds, inBuf: make([]byte, ds.InSize()), }, nil } // Read decompresses bytes into p. Returns io.EOF after the footer has been // validated; returns io.ErrUnexpectedEOF if the underlying reader is drained // before the footer is reached. func (r *Reader) Read(p []byte) (int, error) { if r.err != nil { return 0, r.err } if r.ds == nil { return 0, ErrNullInput } if len(p) == 0 { return 0, nil } for { if r.ds.Finished() { r.err = io.EOF return 0, io.EOF } // Try to decompress whatever is currently buffered (or drain mode // when src is at EOF). if r.inPos < r.inLen || r.eof { consumed, produced, derr := r.ds.Decompress(p, r.inBuf[r.inPos:r.inLen]) if derr != nil { r.err = derr return 0, derr } r.inPos += consumed if produced > 0 { return produced, nil } if consumed == 0 { // No forward progress with current input. if r.eof { if r.ds.Finished() { r.err = io.EOF return 0, io.EOF } r.err = io.ErrUnexpectedEOF return 0, io.ErrUnexpectedEOF } // fall through to refill } else { // Bytes were consumed but no output yet — loop and try again. continue } } // Refill: shift consumed bytes out, then read more. if r.inPos > 0 { r.inLen = copy(r.inBuf, r.inBuf[r.inPos:r.inLen]) r.inPos = 0 } n, rerr := r.src.Read(r.inBuf[r.inLen:]) r.inLen += n if rerr == io.EOF { r.eof = true } else if rerr != nil { r.err = rerr return 0, rerr } if n == 0 && !r.eof { // Underlying reader returned (0, nil): treat as a benign retry by // returning to the caller with no output. return 0, nil } } } // Close releases the underlying [DStream]. The wrapped io.Reader is NOT // closed. Safe to call multiple times. func (r *Reader) Close() error { if r.ds == nil { return nil } err := r.ds.Close() r.ds = nil return err } zxc-0.11.0/wrappers/go/zxc_stream.go000066400000000000000000000146371520102567100173650ustar00rootroot00000000000000/* ZXC - High-performance lossless compression Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. SPDX-License-Identifier: BSD-3-Clause */ package zxc /* #include "zxc.h" */ import "C" import ( "runtime" "unsafe" ) // ============================================================================ // Push Streaming API (single-threaded, caller-driven) // ============================================================================ // CStream is a push-based, single-threaded compression stream — the Go // counterpart of the C [zxc_cstream]. Use it to integrate ZXC into event // loops, callback-driven libraries, or non-blocking network protocols where // the [CompressFile] FILE*-based pipeline is not appropriate. // // CStream is not safe for concurrent use; one goroutine per stream. type CStream struct { ptr *C.zxc_cstream } // NewCStream creates a push compression stream. // // Honoured options: [WithLevel], [WithChecksum]. [WithThreads] and // [WithSeekable] are ignored (the push API is single-threaded and never // emits a seek table). func NewCStream(opts ...Option) (*CStream, error) { o := applyOptions(opts) var copts C.zxc_compress_opts_t copts.level = C.int(o.level) if o.checksum { copts.checksum_enabled = 1 } ptr := C.zxc_cstream_create(&copts) if ptr == nil { return nil, ErrMemory } return &CStream{ptr: ptr}, nil } // Close releases the stream and all internal buffers. Safe to call multiple // times. func (c *CStream) Close() error { if c == nil || c.ptr == nil { return nil } C.zxc_cstream_free(c.ptr) c.ptr = nil return nil } // InSize returns the suggested input chunk size for best throughput. func (c *CStream) InSize() int { if c == nil || c.ptr == nil { return 0 } return int(C.zxc_cstream_in_size(c.ptr)) } // OutSize returns the suggested output chunk size to never trigger a partial // drain. func (c *CStream) OutSize() int { if c == nil || c.ptr == nil { return 0 } return int(C.zxc_cstream_out_size(c.ptr)) } // Compress pushes bytes from in into the stream and writes compressed bytes // to out. Returns: // // - consumed: bytes read from in. // - produced: bytes written into out. // - pending: bytes still staged inside the stream (drain out and call // again with the same or new in to continue). // // A return of (consumed, produced, 0, nil) means in was fully consumed and // no compressed bytes remain pending. Calling with an empty in is valid // (drain-only mode). func (c *CStream) Compress(out, in []byte) (consumed, produced int, pending int64, err error) { if c == nil || c.ptr == nil { return 0, 0, 0, ErrNullInput } var pinner runtime.Pinner defer pinner.Unpin() var inBuf C.zxc_inbuf_t if len(in) > 0 { pinner.Pin(&in[0]) inBuf.src = unsafe.Pointer(&in[0]) } inBuf.size = C.size_t(len(in)) var outBuf C.zxc_outbuf_t if len(out) > 0 { pinner.Pin(&out[0]) outBuf.dst = unsafe.Pointer(&out[0]) } outBuf.size = C.size_t(len(out)) r := C.zxc_cstream_compress(c.ptr, &outBuf, &inBuf) if r < 0 { return 0, 0, 0, errorFromCode(r) } return int(inBuf.pos), int(outBuf.pos), int64(r), nil } // End finalises the stream by flushing the residual block, then writing the // EOF block and the file footer to out. Call repeatedly while pending > 0, // draining out between calls. // // After End returns (produced, 0, nil), the stream is in DONE state; further // calls return ErrNullInput. Always call [CStream.Close] to release the // native resources. func (c *CStream) End(out []byte) (produced int, pending int64, err error) { if c == nil || c.ptr == nil { return 0, 0, ErrNullInput } var pinner runtime.Pinner defer pinner.Unpin() var outBuf C.zxc_outbuf_t if len(out) > 0 { pinner.Pin(&out[0]) outBuf.dst = unsafe.Pointer(&out[0]) } outBuf.size = C.size_t(len(out)) r := C.zxc_cstream_end(c.ptr, &outBuf) if r < 0 { return 0, 0, errorFromCode(r) } return int(outBuf.pos), int64(r), nil } // DStream is a push-based, single-threaded decompression stream — the Go // counterpart of the C [zxc_dstream]. type DStream struct { ptr *C.zxc_dstream } // NewDStream creates a push decompression stream. // // Honoured options: [WithChecksum]. [WithThreads] is ignored. func NewDStream(opts ...Option) (*DStream, error) { o := applyOptions(opts) var dopts C.zxc_decompress_opts_t if o.checksum { dopts.checksum_enabled = 1 } ptr := C.zxc_dstream_create(&dopts) if ptr == nil { return nil, ErrMemory } return &DStream{ptr: ptr}, nil } // Close releases the stream and all internal buffers. Safe to call multiple // times. func (d *DStream) Close() error { if d == nil || d.ptr == nil { return nil } C.zxc_dstream_free(d.ptr) d.ptr = nil return nil } // InSize returns the suggested input chunk size for the decompressor. func (d *DStream) InSize() int { if d == nil || d.ptr == nil { return 0 } return int(C.zxc_dstream_in_size(d.ptr)) } // OutSize returns the suggested output chunk size for the decompressor. func (d *DStream) OutSize() int { if d == nil || d.ptr == nil { return 0 } return int(C.zxc_dstream_out_size(d.ptr)) } // Finished reports whether the decoder has reached and validated the file // footer. Useful to detect truncated streams: if the input source is // drained and Finished returns false, the stream ended prematurely. func (d *DStream) Finished() bool { if d == nil || d.ptr == nil { return false } return C.zxc_dstream_finished(d.ptr) != 0 } // Decompress pushes compressed bytes from in into the stream and writes // decompressed bytes to out. Returns: // // - consumed: bytes read from in. // - produced: bytes written into out. // // A return of (0, 0, nil) when in is non-empty means no progress could be // made: either the parser is waiting for more input (feed more), or the // stream has reached DONE state (any trailing bytes in in are ignored). // Use [DStream.Finished] to disambiguate. func (d *DStream) Decompress(out, in []byte) (consumed, produced int, err error) { if d == nil || d.ptr == nil { return 0, 0, ErrNullInput } var pinner runtime.Pinner defer pinner.Unpin() var inBuf C.zxc_inbuf_t if len(in) > 0 { pinner.Pin(&in[0]) inBuf.src = unsafe.Pointer(&in[0]) } inBuf.size = C.size_t(len(in)) var outBuf C.zxc_outbuf_t if len(out) > 0 { pinner.Pin(&out[0]) outBuf.dst = unsafe.Pointer(&out[0]) } outBuf.size = C.size_t(len(out)) r := C.zxc_dstream_decompress(d.ptr, &outBuf, &inBuf) if r < 0 { return 0, 0, errorFromCode(r) } return int(inBuf.pos), int(outBuf.pos), nil } zxc-0.11.0/wrappers/go/zxc_test.go000066400000000000000000000406201520102567100170400ustar00rootroot00000000000000/* ZXC - High-performance lossless compression Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. SPDX-License-Identifier: BSD-3-Clause */ package zxc import ( "bytes" "fmt" "io" "os" "path/filepath" "testing" ) // ============================================================================ // Buffer API Tests // ============================================================================ func TestRoundtrip(t *testing.T) { data := []byte("Hello, ZXC! This is a test of the Go wrapper.") compressed, err := Compress(data) if err != nil { t.Fatalf("Compress: %v", err) } decompressed, err := Decompress(compressed) if err != nil { t.Fatalf("Decompress: %v", err) } if !bytes.Equal(data, decompressed) { t.Fatal("roundtrip mismatch") } } func TestAllLevels(t *testing.T) { data := []byte("Test data with repetition: CCCCCCCCCCCCCCCCCCCCCCCCCCCCCC") for _, level := range AllLevels() { compressed, err := Compress(data, WithLevel(level)) if err != nil { t.Fatalf("Compress at level %d: %v", level, err) } decompressed, err := Decompress(compressed) if err != nil { t.Fatalf("Decompress at level %d: %v", level, err) } if !bytes.Equal(data, decompressed) { t.Fatalf("roundtrip mismatch at level %d", level) } } } func TestLargeData(t *testing.T) { // 1 MB of compressible data data := make([]byte, 1024*1024) for i := range data { data[i] = byte((i % 256) ^ ((i / 256) % 256)) } compressed, err := Compress(data) if err != nil { t.Fatalf("Compress: %v", err) } if len(compressed) >= len(data) { t.Fatalf("expected compression, got %d >= %d", len(compressed), len(data)) } decompressed, err := Decompress(compressed) if err != nil { t.Fatalf("Decompress: %v", err) } if !bytes.Equal(data, decompressed) { t.Fatal("large data roundtrip mismatch") } } func TestDecompressedSize(t *testing.T) { data := []byte("Hello, world! Testing DecompressedSize function.") compressed, err := Compress(data) if err != nil { t.Fatalf("Compress: %v", err) } size, err := DecompressedSize(compressed) if err != nil { t.Fatalf("DecompressedSize: %v", err) } if size != uint64(len(data)) { t.Fatalf("DecompressedSize = %d, want %d", size, len(data)) } } func TestCompressBound(t *testing.T) { bound := CompressBound(1024) if bound <= 1024 { t.Fatalf("CompressBound(1024) = %d, want > 1024", bound) } } func TestInvalidData(t *testing.T) { _, err := Decompress([]byte("not valid zxc data")) if err == nil { t.Fatal("expected error on invalid data") } } func TestEmptyInput(t *testing.T) { _, err := Compress(nil) if err == nil { t.Fatal("expected error on nil input") } _, err = Compress([]byte{}) if err == nil { t.Fatal("expected error on empty input") } } func TestChecksumRoundtrip(t *testing.T) { data := []byte("Test with checksum enabled for data integrity verification") compressed, err := Compress(data, WithChecksum(true)) if err != nil { t.Fatalf("Compress with checksum: %v", err) } decompressed, err := Decompress(compressed, WithChecksum(true)) if err != nil { t.Fatalf("Decompress with checksum: %v", err) } if !bytes.Equal(data, decompressed) { t.Fatal("checksum roundtrip mismatch") } } func TestCompressTo(t *testing.T) { data := []byte("Testing CompressTo with pre-allocated buffer") output := make([]byte, CompressBound(len(data))) n, err := CompressTo(data, output) if err != nil { t.Fatalf("CompressTo: %v", err) } decompressed, err := Decompress(output[:n]) if err != nil { t.Fatalf("Decompress: %v", err) } if !bytes.Equal(data, decompressed) { t.Fatal("CompressTo roundtrip mismatch") } } func TestVersion(t *testing.T) { major, minor, patch := Version() s := VersionString() if s == "" { t.Fatal("VersionString returned empty") } expected := fmt.Sprintf("%d.%d.%d", major, minor, patch) if s != expected { t.Fatalf("VersionString() = %q, want %q", s, expected) } t.Logf("ZXC version: %s", s) } func TestErrorMessages(t *testing.T) { errors := []*Error{ ErrMemory, ErrDstTooSmall, ErrCorruptData, ErrBadChecksum, } for _, e := range errors { msg := e.Error() if msg == "" { t.Fatalf("expected non-empty error message for code %d", e.Code) } } } // ============================================================================ // Streaming API Tests // ============================================================================ func tempPath(t *testing.T, name string) string { t.Helper() dir := filepath.Join(os.TempDir(), "zxc_go_test") if err := os.MkdirAll(dir, 0o755); err != nil { t.Fatalf("MkdirAll: %v", err) } return filepath.Join(dir, name) } func TestFileRoundtrip(t *testing.T) { inputPath := tempPath(t, "roundtrip_input.bin") compressedPath := tempPath(t, "roundtrip_compressed.zxc") outputPath := tempPath(t, "roundtrip_output.bin") defer os.Remove(inputPath) defer os.Remove(compressedPath) defer os.Remove(outputPath) // 64 KB of compressible data data := make([]byte, 64*1024) for i := range data { data[i] = byte((i % 256) ^ ((i / 256) % 256)) } if err := os.WriteFile(inputPath, data, 0o644); err != nil { t.Fatalf("WriteFile: %v", err) } csize, err := CompressFile(inputPath, compressedPath) if err != nil { t.Fatalf("CompressFile: %v", err) } if csize <= 0 { t.Fatalf("CompressFile returned %d bytes", csize) } dsize, err := DecompressFile(compressedPath, outputPath) if err != nil { t.Fatalf("DecompressFile: %v", err) } if dsize != int64(len(data)) { t.Fatalf("DecompressFile: got %d bytes, want %d", dsize, len(data)) } result, err := os.ReadFile(outputPath) if err != nil { t.Fatalf("ReadFile: %v", err) } if !bytes.Equal(data, result) { t.Fatal("file roundtrip mismatch") } } func TestFileAllLevels(t *testing.T) { inputPath := tempPath(t, "levels_input.bin") defer os.Remove(inputPath) data := make([]byte, 32*1024) for i := range data { data[i] = byte((i % 256) ^ ((i / 256) % 256)) } if err := os.WriteFile(inputPath, data, 0o644); err != nil { t.Fatalf("WriteFile: %v", err) } for _, level := range AllLevels() { compressedPath := tempPath(t, "levels_compressed.zxc") outputPath := tempPath(t, "levels_output.bin") _, err := CompressFile(inputPath, compressedPath, WithLevel(level)) if err != nil { t.Fatalf("CompressFile at level %d: %v", level, err) } _, err = DecompressFile(compressedPath, outputPath) if err != nil { t.Fatalf("DecompressFile at level %d: %v", level, err) } result, err := os.ReadFile(outputPath) if err != nil { t.Fatalf("ReadFile: %v", err) } if !bytes.Equal(data, result) { t.Fatalf("file roundtrip mismatch at level %d", level) } os.Remove(compressedPath) os.Remove(outputPath) } } func TestFileMultithreaded(t *testing.T) { inputPath := tempPath(t, "mt_input.bin") compressedPath := tempPath(t, "mt_compressed.zxc") outputPath := tempPath(t, "mt_output.bin") defer os.Remove(inputPath) defer os.Remove(compressedPath) defer os.Remove(outputPath) // 1 MB of data data := make([]byte, 1024*1024) for i := range data { data[i] = byte((i % 256) ^ ((i / 256) % 256)) } if err := os.WriteFile(inputPath, data, 0o644); err != nil { t.Fatalf("WriteFile: %v", err) } for _, threads := range []int{1, 2, 4} { _, err := CompressFile(inputPath, compressedPath, WithThreads(threads)) if err != nil { t.Fatalf("CompressFile with %d threads: %v", threads, err) } dsize, err := DecompressFile(compressedPath, outputPath, WithThreads(threads)) if err != nil { t.Fatalf("DecompressFile with %d threads: %v", threads, err) } if dsize != int64(len(data)) { t.Fatalf("got %d bytes with %d threads, want %d", dsize, threads, len(data)) } result, err := os.ReadFile(outputPath) if err != nil { t.Fatalf("ReadFile: %v", err) } if !bytes.Equal(data, result) { t.Fatalf("mismatch with %d threads", threads) } } } // ============================================================================ // Push Streaming API Tests // ============================================================================ // pstreamRoundtrip drives a CStream / DStream pair end-to-end and returns // the decoded bytes. func pstreamRoundtrip(t *testing.T, data []byte, copts, dopts []Option) []byte { t.Helper() cs, err := NewCStream(copts...) if err != nil { t.Fatalf("NewCStream: %v", err) } defer cs.Close() outCap := cs.OutSize() if outCap < 64 { outCap = 64 } out := make([]byte, outCap) var compressed bytes.Buffer cursor := 0 for cursor < len(data) { consumed, produced, _, err := cs.Compress(out, data[cursor:]) if err != nil { t.Fatalf("CStream.Compress: %v", err) } cursor += consumed compressed.Write(out[:produced]) } for { produced, pending, err := cs.End(out) if err != nil { t.Fatalf("CStream.End: %v", err) } compressed.Write(out[:produced]) if pending == 0 { break } } ds, err := NewDStream(dopts...) if err != nil { t.Fatalf("NewDStream: %v", err) } defer ds.Close() dout := make([]byte, 64*1024) var decompressed bytes.Buffer src := compressed.Bytes() cursor = 0 for cursor < len(src) && !ds.Finished() { consumed, produced, err := ds.Decompress(dout, src[cursor:]) if err != nil { t.Fatalf("DStream.Decompress: %v", err) } cursor += consumed decompressed.Write(dout[:produced]) if consumed == 0 && produced == 0 { break } } for { _, produced, err := ds.Decompress(dout, nil) if err != nil { t.Fatalf("DStream.Decompress (drain): %v", err) } decompressed.Write(dout[:produced]) if produced == 0 { break } } if !ds.Finished() { t.Fatalf("DStream did not finalise (truncated stream?)") } return decompressed.Bytes() } func TestPStreamSmallRoundtrip(t *testing.T) { data := []byte("Hello pstream! Round-trip through the Go push API.") got := pstreamRoundtrip(t, data, nil, nil) if !bytes.Equal(got, data) { t.Fatalf("mismatch: got %d bytes, want %d", len(got), len(data)) } } func TestPStreamWithChecksum(t *testing.T) { data := make([]byte, 32*1024) for i := range data { data[i] = byte(i % 251) } copts := []Option{WithChecksum(true)} dopts := []Option{WithChecksum(true)} got := pstreamRoundtrip(t, data, copts, dopts) if !bytes.Equal(got, data) { t.Fatalf("checksum roundtrip mismatch (%d vs %d bytes)", len(got), len(data)) } } func TestPStreamMultiBlock(t *testing.T) { data := make([]byte, 512*1024) for i := range data { data[i] = byte((i * 7) % 256) } got := pstreamRoundtrip(t, data, nil, nil) if !bytes.Equal(got, data) { t.Fatalf("multi-block roundtrip mismatch (%d vs %d bytes)", len(got), len(data)) } } func TestPStreamSizeHints(t *testing.T) { cs, err := NewCStream() if err != nil { t.Fatalf("NewCStream: %v", err) } defer cs.Close() if cs.InSize() == 0 || cs.OutSize() == 0 { t.Fatalf("cstream size hints should be non-zero") } ds, err := NewDStream() if err != nil { t.Fatalf("NewDStream: %v", err) } defer ds.Close() if ds.InSize() == 0 || ds.OutSize() == 0 { t.Fatalf("dstream size hints should be non-zero") } if ds.Finished() { t.Fatalf("dstream should not be finished before input") } } // ============================================================================ // io.Reader / io.Writer adapter tests // ============================================================================ func ioRoundtrip(t *testing.T, data []byte, copts, dopts []Option) []byte { t.Helper() var buf bytes.Buffer w, err := NewWriter(&buf, copts...) if err != nil { t.Fatalf("NewWriter: %v", err) } n, err := w.Write(data) if err != nil { t.Fatalf("Writer.Write: %v", err) } if n != len(data) { t.Fatalf("Writer.Write short: %d/%d", n, len(data)) } if err := w.Close(); err != nil { t.Fatalf("Writer.Close: %v", err) } r, err := NewReader(&buf, dopts...) if err != nil { t.Fatalf("NewReader: %v", err) } defer r.Close() got, err := io.ReadAll(r) if err != nil { t.Fatalf("ReadAll: %v", err) } return got } func TestWriterReaderSmallRoundtrip(t *testing.T) { data := []byte("Hello io.Writer / io.Reader bridge over ZXC.") got := ioRoundtrip(t, data, nil, nil) if !bytes.Equal(got, data) { t.Fatalf("mismatch: got %q want %q", got, data) } } func TestWriterReaderLargeRoundtrip(t *testing.T) { data := make([]byte, 2*1024*1024) for i := range data { data[i] = byte((i * 13) % 251) } got := ioRoundtrip(t, data, nil, nil) if !bytes.Equal(got, data) { t.Fatalf("large roundtrip mismatch (%d vs %d)", len(got), len(data)) } } func TestWriterReaderManySmallWrites(t *testing.T) { var buf bytes.Buffer w, err := NewWriter(&buf) if err != nil { t.Fatalf("NewWriter: %v", err) } want := make([]byte, 0, 64*1024) for i := 0; i < 4096; i++ { chunk := []byte{byte(i), byte(i >> 8), byte(i ^ 0x5A)} want = append(want, chunk...) if _, err := w.Write(chunk); err != nil { t.Fatalf("Write(%d): %v", i, err) } } if err := w.Close(); err != nil { t.Fatalf("Writer.Close: %v", err) } r, err := NewReader(&buf) if err != nil { t.Fatalf("NewReader: %v", err) } defer r.Close() got, err := io.ReadAll(r) if err != nil { t.Fatalf("ReadAll: %v", err) } if !bytes.Equal(got, want) { t.Fatalf("many-writes mismatch (%d vs %d)", len(got), len(want)) } } func TestWriterReaderWithChecksum(t *testing.T) { data := make([]byte, 32*1024) for i := range data { data[i] = byte(i) } got := ioRoundtrip(t, data, []Option{WithChecksum(true)}, []Option{WithChecksum(true)}) if !bytes.Equal(got, data) { t.Fatalf("checksum roundtrip mismatch") } } func TestWriterCloseIsIdempotent(t *testing.T) { var buf bytes.Buffer w, err := NewWriter(&buf) if err != nil { t.Fatalf("NewWriter: %v", err) } if _, err := w.Write([]byte("hello")); err != nil { t.Fatalf("Write: %v", err) } if err := w.Close(); err != nil { t.Fatalf("Close 1: %v", err) } if err := w.Close(); err != nil { t.Fatalf("Close 2: %v", err) } } func TestReaderCloseIsIdempotent(t *testing.T) { var buf bytes.Buffer w, _ := NewWriter(&buf) w.Write([]byte("xxx")) w.Close() r, err := NewReader(&buf) if err != nil { t.Fatalf("NewReader: %v", err) } if err := r.Close(); err != nil { t.Fatalf("Close 1: %v", err) } if err := r.Close(); err != nil { t.Fatalf("Close 2: %v", err) } } func TestReaderTruncatedFrame(t *testing.T) { var buf bytes.Buffer w, _ := NewWriter(&buf) payload := bytes.Repeat([]byte("ABCDEFGH"), 4096) if _, err := w.Write(payload); err != nil { t.Fatalf("Write: %v", err) } if err := w.Close(); err != nil { t.Fatalf("Close: %v", err) } // Truncate before the footer to force a premature EOF. full := buf.Bytes() truncated := full[:len(full)/2] r, err := NewReader(bytes.NewReader(truncated)) if err != nil { t.Fatalf("NewReader: %v", err) } defer r.Close() _, err = io.ReadAll(r) if err == nil { t.Fatal("expected error reading truncated frame, got nil") } } // ============================================================================ // DetectZxc // ============================================================================ func TestDetectZxc(t *testing.T) { // Real ZXC frame must be detected. compressed, err := Compress([]byte("magic-detection-input")) if err != nil { t.Fatalf("Compress: %v", err) } if !DetectZxc(compressed) { t.Fatal("DetectZxc returned false for a valid ZXC frame") } // Same goes for an io.Writer-produced frame. var buf bytes.Buffer w, _ := NewWriter(&buf) w.Write([]byte("hi")) w.Close() if !DetectZxc(buf.Bytes()) { t.Fatal("DetectZxc returned false for an io.Writer-produced frame") } // Negative cases. cases := [][]byte{ nil, {}, {0xF5, 0x2E, 0xB0}, // too short {0x00, 0x00, 0x00, 0x00}, []byte("not a zxc frame at all"), } for _, c := range cases { if DetectZxc(c) { t.Fatalf("DetectZxc returned true for non-ZXC input %x", c) } } } // ============================================================================ // Benchmarks // ============================================================================ func BenchmarkCompress(b *testing.B) { data := make([]byte, 1024*1024) for i := range data { data[i] = byte((i % 256) ^ ((i / 256) % 256)) } b.SetBytes(int64(len(data))) b.ResetTimer() for i := 0; i < b.N; i++ { _, err := Compress(data) if err != nil { b.Fatal(err) } } } func BenchmarkDecompress(b *testing.B) { data := make([]byte, 1024*1024) for i := range data { data[i] = byte((i % 256) ^ ((i / 256) % 256)) } compressed, err := Compress(data) if err != nil { b.Fatal(err) } b.SetBytes(int64(len(data))) b.ResetTimer() for i := 0; i < b.N; i++ { _, err := Decompress(compressed) if err != nil { b.Fatal(err) } } } zxc-0.11.0/wrappers/nodejs/000077500000000000000000000000001520102567100155215ustar00rootroot00000000000000zxc-0.11.0/wrappers/nodejs/.gitignore000066400000000000000000000000711520102567100175070ustar00rootroot00000000000000node_modules/ build/ prebuilds/ *.node package-lock.json zxc-0.11.0/wrappers/nodejs/.nvmrc000066400000000000000000000000021520102567100166370ustar00rootroot0000000000000022zxc-0.11.0/wrappers/nodejs/CMakeLists.txt000066400000000000000000000027741520102567100202730ustar00rootroot00000000000000cmake_minimum_required(VERSION 3.14) project(zxc_nodejs C CXX) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) # ---- Core ZXC library ---- add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../../ ${CMAKE_CURRENT_BINARY_DIR}/zxc_core_build) # ---- Node.js addon ---- include_directories(${CMAKE_JS_INC}) file(GLOB SOURCE_FILES "src/zxc_addon.cc") add_library(${PROJECT_NAME} SHARED ${SOURCE_FILES} ${CMAKE_JS_SRC}) set_target_properties(${PROJECT_NAME} PROPERTIES PREFIX "" SUFFIX ".node" ) target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../include ${CMAKE_JS_INC} ) # Generate node.lib on Windows (cmake-js provides .def but not .lib) if(MSVC AND CMAKE_JS_NODELIB_DEF AND CMAKE_JS_NODELIB_TARGET) execute_process(COMMAND ${CMAKE_AR} /def:${CMAKE_JS_NODELIB_DEF} /out:${CMAKE_JS_NODELIB_TARGET} ${CMAKE_STATIC_LINKER_FLAGS}) endif() target_link_libraries(${PROJECT_NAME} PRIVATE zxc_lib ${CMAKE_JS_LIB} ) # node-addon-api (header-only, no exceptions by default) execute_process( COMMAND node -p "require('node-addon-api').include" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE NAPI_INCLUDE_DIR OUTPUT_STRIP_TRAILING_WHITESPACE ) # Strip surrounding quotes if present string(REPLACE "\"" "" NAPI_INCLUDE_DIR ${NAPI_INCLUDE_DIR}) target_include_directories(${PROJECT_NAME} PRIVATE ${NAPI_INCLUDE_DIR}) target_compile_definitions(${PROJECT_NAME} PRIVATE NAPI_VERSION=8) zxc-0.11.0/wrappers/nodejs/README.md000066400000000000000000000050031520102567100167760ustar00rootroot00000000000000# ZXC Node.js Bindings High-performance Node.js bindings for the **ZXC** asymmetric compressor, optimized for **fast decompression**. Designed for *Write Once, Read Many* workloads like ML datasets, game assets, and caches. ## Features - **Blazing fast decompression** - ZXC is specifically optimized for read-heavy workloads. - **Native N-API addon** - compiled C code via cmake-js for maximum performance. - **Buffer support** - works directly with Node.js `Buffer` objects. - **TypeScript declarations** - full type definitions included. ## Installation (from source) ```bash git clone https://github.com/hellobertrand/zxc.git cd zxc/wrappers/nodejs npm install ``` ### Prerequisites - Node.js >= 16.0.0 - CMake >= 3.14 - A C17/C++17 compiler (GCC, Clang, or MSVC) ## Usage ```javascript const zxc = require('zxc-compress'); // Compress const data = Buffer.from('Hello, World!'.repeat(1_000)); const compressed = zxc.compress(data, { level: zxc.LEVEL_DEFAULT }); // Decompress (auto-detects size) const decompressed = zxc.decompress(compressed); console.log(`Original: ${data.length} bytes`); console.log(`Compressed: ${compressed.length} bytes`); console.log(`Ratio: ${(compressed.length / data.length * 100).toFixed(1)}%`); ``` ## API ### `compress(data, options?)` Compress a Buffer. | Parameter | Type | Default | Description | |-----------|------|---------|-------------| | `data` | `Buffer` | - | Input data | | `options.level` | `number` | `LEVEL_DEFAULT` | Compression level (1–5) | | `options.checksum` | `boolean` | `false` | Enable checksum | Returns: `Buffer` - compressed data. ### `decompress(data, options?)` Decompress a ZXC compressed Buffer. | Parameter | Type | Default | Description | |-----------|------|---------|-------------| | `data` | `Buffer` | - | Compressed data | | `options.size` | `number` | auto | Expected decompressed size | | `options.checksum` | `boolean` | `false` | Verify checksum | Returns: `Buffer` - decompressed data. ### `compressBound(inputSize)` Returns the maximum compressed size for a given input size. ### `getDecompressedSize(data)` Returns the original size from a ZXC compressed buffer (reads footer only). ### Constants | Constant | Value | Description | |----------|-------|-------------| | `LEVEL_FASTEST` | 1 | Fastest compression | | `LEVEL_FAST` | 2 | Fast compression | | `LEVEL_DEFAULT` | 3 | Recommended balance | | `LEVEL_BALANCED` | 4 | Good ratio, good speed | | `LEVEL_COMPACT` | 5 | Highest density | ## Testing ```bash npm test ``` ## License BSD-3-Clause zxc-0.11.0/wrappers/nodejs/lib/000077500000000000000000000000001520102567100162675ustar00rootroot00000000000000zxc-0.11.0/wrappers/nodejs/lib/index.d.ts000066400000000000000000000147251520102567100202010ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /** Fastest compression, best for real-time applications. */ export const LEVEL_FASTEST: number; /** Fast compression, good for real-time applications. */ export const LEVEL_FAST: number; /** Recommended: ratio > LZ4, decode speed > LZ4. */ export const LEVEL_DEFAULT: number; /** Good ratio, good decode speed. */ export const LEVEL_BALANCED: number; /** High density. Best for storage/firmware/assets. */ export const LEVEL_COMPACT: number; /** Maximum density: Huffman-coded literals on top of COMPACT. */ export const LEVEL_DENSITY: number; /** Memory allocation failure. */ export const ERROR_MEMORY: number; /** Destination buffer too small. */ export const ERROR_DST_TOO_SMALL: number; /** Source buffer too small or truncated input. */ export const ERROR_SRC_TOO_SMALL: number; /** Invalid magic word in file header. */ export const ERROR_BAD_MAGIC: number; /** Unsupported file format version. */ export const ERROR_BAD_VERSION: number; /** Corrupted or invalid header (CRC mismatch). */ export const ERROR_BAD_HEADER: number; /** Block or global checksum verification failed. */ export const ERROR_BAD_CHECKSUM: number; /** Corrupted compressed data. */ export const ERROR_CORRUPT_DATA: number; /** Invalid match offset during decompression. */ export const ERROR_BAD_OFFSET: number; /** Buffer overflow detected during processing. */ export const ERROR_OVERFLOW: number; /** Read/write/seek failure on file. */ export const ERROR_IO: number; /** Required input pointer is NULL. */ export const ERROR_NULL_INPUT: number; /** Unknown or unexpected block type. */ export const ERROR_BAD_BLOCK_TYPE: number; /** Invalid block size. */ export const ERROR_BAD_BLOCK_SIZE: number; export interface CompressOptions { /** Compression level (1-6). Defaults to LEVEL_DEFAULT. */ level?: number; /** Enable checksum verification. Defaults to false. */ checksum?: boolean; /** Enable seek table for random-access decompression. Defaults to false. */ seekable?: boolean; } export interface DecompressOptions { /** Expected decompressed size. If omitted, read from header. */ size?: number; /** Enable checksum verification. Defaults to false. */ checksum?: boolean; } /** * Returns the maximum compressed size for a given input size. * Useful for pre-allocating output buffers. */ export function compressBound(inputSize: number): number; /** * Compress a Buffer using the ZXC algorithm. */ export function compress(data: Buffer, options?: CompressOptions): Buffer; /** * Returns the original decompressed size from a ZXC compressed buffer. * Reads the footer without performing decompression. */ export function getDecompressedSize(data: Buffer): number; /** * Decompress a ZXC compressed Buffer. */ export function decompress(data: Buffer, options?: DecompressOptions): Buffer; /** * Returns a human-readable name for a given error code. */ export function errorName(code: number): string; /** Returns the minimum supported compression level (currently 1). */ export function minLevel(): number; /** Returns the maximum supported compression level (currently 6). */ export function maxLevel(): number; /** Returns the default compression level (currently 3). */ export function defaultLevel(): number; /** * Returns the version string reported by the linked native libzxc * (e.g. "0.11.0"). Distinct from the npm package version. */ export function libraryVersion(): string; export interface CStreamOptions { /** Compression level (1-6). Defaults to LEVEL_DEFAULT. */ level?: number; /** Enable per-block and global checksums. Defaults to false. */ checksum?: boolean; /** Block size in bytes (0 = default 512 KB). Power of 2, 4 KB – 2 MB. */ blockSize?: number; } /** * Push-based, single-threaded compression stream. * * The Node.js counterpart of the C `zxc_cstream`. Each call to * `compress(buf)` returns the compressed bytes produced from that input * (may be empty if the bytes fit in the internal block accumulator). * Always call `end()` to flush the residual block, EOF marker and footer. */ export class CStream { constructor(options?: CStreamOptions); /** Push input and return any compressed bytes produced this call. */ compress(data: Buffer): Buffer; /** Finalise the stream: residual block + EOF + file footer. */ end(): Buffer; /** Release native resources. Idempotent. */ close(): void; /** Suggested input chunk size in bytes. */ inSize(): number; /** Suggested output chunk size in bytes. */ outSize(): number; } export interface DStreamOptions { /** Verify per-block and global checksums when present. Defaults to false. */ checksum?: boolean; } /** * Push-based, single-threaded decompression stream. */ export class DStream { constructor(options?: DStreamOptions); /** Push compressed bytes and return any decompressed bytes produced. */ decompress(data: Buffer): Buffer; /** True once the decoder has reached and validated the file footer. */ finished(): boolean; /** Release native resources. Idempotent. */ close(): void; /** Suggested input chunk size in bytes. */ inSize(): number; /** Suggested output chunk size in bytes. */ outSize(): number; } // ---------- stream.Transform adapters ---------- import { Transform, TransformOptions } from 'node:stream'; /** Returns true if `buf` starts with the ZXC file magic word. */ export function detectZxc(buf: Buffer | Uint8Array): boolean; export interface CompressStreamOptions extends TransformOptions, CStreamOptions {} export interface DecompressStreamOptions extends TransformOptions, DStreamOptions {} /** * `stream.Transform` that compresses bytes through a ZXC frame. Designed to * be piped between any Node Readable/Writable (fs, http, tar-stream, OCI * registry clients, etc.). Mirrors `zlib.createGzip()` ergonomics. */ export class CompressStream extends Transform { constructor(options?: CompressStreamOptions); } /** * `stream.Transform` that decompresses a ZXC frame. Emits `'error'` with * `code === 'ZXC_TRUNCATED'` if the input ends before the footer. */ export class DecompressStream extends Transform { constructor(options?: DecompressStreamOptions); } export function createCompressStream(options?: CompressStreamOptions): CompressStream; export function createDecompressStream(options?: DecompressStreamOptions): DecompressStream; zxc-0.11.0/wrappers/nodejs/lib/index.js000066400000000000000000000261451520102567100177440ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ 'use strict'; const { Transform } = require('node:stream'); const native = require('../build/Release/zxc_nodejs.node'); // Re-export compression level constants const LEVEL_FASTEST = native.LEVEL_FASTEST; const LEVEL_FAST = native.LEVEL_FAST; const LEVEL_DEFAULT = native.LEVEL_DEFAULT; const LEVEL_BALANCED = native.LEVEL_BALANCED; const LEVEL_COMPACT = native.LEVEL_COMPACT; const LEVEL_DENSITY = native.LEVEL_DENSITY; // Re-export error constants const ERROR_MEMORY = native.ERROR_MEMORY; const ERROR_DST_TOO_SMALL = native.ERROR_DST_TOO_SMALL; const ERROR_SRC_TOO_SMALL = native.ERROR_SRC_TOO_SMALL; const ERROR_BAD_MAGIC = native.ERROR_BAD_MAGIC; const ERROR_BAD_VERSION = native.ERROR_BAD_VERSION; const ERROR_BAD_HEADER = native.ERROR_BAD_HEADER; const ERROR_BAD_CHECKSUM = native.ERROR_BAD_CHECKSUM; const ERROR_CORRUPT_DATA = native.ERROR_CORRUPT_DATA; const ERROR_BAD_OFFSET = native.ERROR_BAD_OFFSET; const ERROR_OVERFLOW = native.ERROR_OVERFLOW; const ERROR_IO = native.ERROR_IO; const ERROR_NULL_INPUT = native.ERROR_NULL_INPUT; const ERROR_BAD_BLOCK_TYPE = native.ERROR_BAD_BLOCK_TYPE; const ERROR_BAD_BLOCK_SIZE = native.ERROR_BAD_BLOCK_SIZE; /** * Returns the minimum supported compression level (currently 1). * @returns {number} */ function minLevel() { return native.minLevel(); } /** * Returns the maximum supported compression level (currently 6). * @returns {number} */ function maxLevel() { return native.maxLevel(); } /** * Returns the default compression level (currently 3). * @returns {number} */ function defaultLevel() { return native.defaultLevel(); } /** * Returns the version string reported by the linked native libzxc * (e.g. "0.11.0"). Distinct from the npm package version. * @returns {string} */ function libraryVersion() { return native.libraryVersion(); } /** * Returns a human-readable name for a given error code. * * @param {number} code - ZXC error code. * @returns {string} Error name (e.g., "ZXC_ERROR_DST_TOO_SMALL"). */ function errorName(code) { if (typeof code !== 'number') { throw new TypeError('code must be a number'); } return native.errorName(code); } /** * Returns the maximum compressed size for a given input size. * Useful for pre-allocating output buffers. * * @param {number} inputSize - Size of the input data in bytes. * @returns {number} Maximum required buffer size in bytes. */ function compressBound(inputSize) { if (typeof inputSize !== 'number' || inputSize < 0) { throw new TypeError('inputSize must be a non-negative number'); } return native.compressBound(inputSize); } /** * Compress a Buffer using the ZXC algorithm. * * @param {Buffer} data - Buffer to compress. * @param {object} [options] - Compression options. * @param {number} [options.level=LEVEL_DEFAULT] - Compression level (1-6). * @param {boolean} [options.checksum=false] - Enable checksum verification. * @param {boolean} [options.seekable=false] - Enable seek table for random-access decompression. * @returns {Buffer} Compressed data. */ function compress(data, options = {}) { if (!Buffer.isBuffer(data)) { throw new TypeError('data must be a Buffer'); } const level = options.level !== undefined ? options.level : LEVEL_DEFAULT; const checksum = options.checksum !== undefined ? options.checksum : false; const seekable = options.seekable !== undefined ? options.seekable : false; if (data.length === 0 && !checksum) { return Buffer.alloc(0); } return native.compress(data, level, checksum, seekable); } /** * Returns the original decompressed size from a ZXC compressed buffer. * Reads the footer without performing decompression. * * @param {Buffer} data - Compressed data buffer. * @returns {number} Original uncompressed size in bytes, or 0 if invalid. */ function getDecompressedSize(data) { if (!Buffer.isBuffer(data)) { throw new TypeError('data must be a Buffer'); } return native.getDecompressedSize(data); } /** * Decompress a ZXC compressed Buffer. * * @param {Buffer} data - Compressed data. * @param {object} [options] - Decompression options. * @param {number} [options.size] - Expected decompressed size. If omitted, read from header. * @param {boolean} [options.checksum=false] - Enable checksum verification. * @returns {Buffer} Decompressed data. */ function decompress(data, options = {}) { if (!Buffer.isBuffer(data)) { throw new TypeError('data must be a Buffer'); } const checksum = options.checksum !== undefined ? options.checksum : false; if (data.length === 0 && !checksum) { return Buffer.alloc(0); } let size = options.size; if (size === undefined) { size = getDecompressedSize(data); if (size === 0) { throw new Error('Invalid ZXC header or data too short to determine size'); } } return native.decompress(data, size, checksum); } /** * Push-based, single-threaded compression stream. * * The Node.js counterpart of the C `zxc_cstream`. Use it when you cannot * block on a `FILE*` (event loops, web frameworks, network protocols). * * @example * const cs = new zxc.CStream({ level: zxc.LEVEL_DEFAULT, checksum: true }); * const chunks = []; * for await (const part of source) chunks.push(cs.compress(part)); * chunks.push(cs.end()); * cs.close(); * sink.write(Buffer.concat(chunks)); */ const CStream = native.CStream; /** * Push-based, single-threaded decompression stream. * * @example * const ds = new zxc.DStream({ checksum: true }); * for await (const part of compressed) sink.write(ds.decompress(part)); * if (!ds.finished()) throw new Error('truncated'); * ds.close(); */ const DStream = native.DStream; // ============================================================================= // stream.Transform adapters over the push streaming API // ============================================================================= /** * Returns true if `buf` starts with the ZXC file magic word. * * Useful for content-type sniffing in containers / object stores that need * to dispatch on media type (e.g. OCI). Cheap and side-effect free; does * not validate the rest of the header or the footer. * * Magic word identifying a ZXC file frame: little-endian 0x9CB02EF5. * * @param {Buffer|Uint8Array} buf * @returns {boolean} */ function detectZxc(buf) { if (!buf || buf.length < 4) return false; return buf[0] === 0xF5 && buf[1] === 0x2E && buf[2] === 0xB0 && buf[3] === 0x9C; } /** * A Node.js `stream.Transform` that compresses bytes through a ZXC frame. * * @example * const fs = require('node:fs'); * const { pipeline } = require('node:stream/promises'); * await pipeline( * fs.createReadStream('input.bin'), * zxc.createCompressStream({ level: zxc.LEVEL_DEFAULT }), * fs.createWriteStream('output.zxc'), * ); */ class CompressStream extends Transform { constructor(options = {}) { const { level, checksum, blockSize, ...transformOpts } = options; super(transformOpts); this._cs = new CStream({ ...(level !== undefined ? { level } : {}), ...(checksum !== undefined ? { checksum } : {}), ...(blockSize !== undefined ? { blockSize } : {}), }); } _transform(chunk, encoding, callback) { try { const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, typeof encoding === 'string' ? encoding : 'utf8'); const out = this._cs.compress(buf); if (out.length > 0) this.push(out); callback(); } catch (err) { callback(err); } } _flush(callback) { try { const tail = this._cs.end(); if (tail.length > 0) this.push(tail); this._cs.close(); callback(); } catch (err) { callback(err); } } _destroy(err, callback) { try { this._cs.close(); } catch (_) { /* idempotent */ } callback(err); } } /** * A Node.js `stream.Transform` that decompresses a ZXC frame. * * Emits `'error'` with code `'ZXC_TRUNCATED'` if the input ends before the * footer is reached. * * @example * const fs = require('node:fs'); * const { pipeline } = require('node:stream/promises'); * await pipeline( * fs.createReadStream('input.zxc'), * zxc.createDecompressStream(), * fs.createWriteStream('output.bin'), * ); */ class DecompressStream extends Transform { constructor(options = {}) { const { checksum, ...transformOpts } = options; super(transformOpts); this._ds = new DStream({ ...(checksum !== undefined ? { checksum } : {}), }); } _transform(chunk, encoding, callback) { try { const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, typeof encoding === 'string' ? encoding : 'utf8'); const out = this._ds.decompress(buf); if (out.length > 0) this.push(out); callback(); } catch (err) { callback(err); } } _flush(callback) { try { if (!this._ds.finished()) { const err = new Error('zxc: input drained before footer (truncated frame)'); err.code = 'ZXC_TRUNCATED'; this._ds.close(); callback(err); return; } this._ds.close(); callback(); } catch (err) { callback(err); } } _destroy(err, callback) { try { this._ds.close(); } catch (_) { /* idempotent */ } callback(err); } } /** * Factory matching the `zlib.createGzip()` convention. * @param {object} [options] * @returns {CompressStream} */ function createCompressStream(options) { return new CompressStream(options); } /** * Factory matching the `zlib.createGunzip()` convention. * @param {object} [options] * @returns {DecompressStream} */ function createDecompressStream(options) { return new DecompressStream(options); } module.exports = { // Functions compress, decompress, compressBound, getDecompressedSize, // Push streaming classes CStream, DStream, // stream.Transform adapters CompressStream, DecompressStream, createCompressStream, createDecompressStream, detectZxc, // Library info helpers minLevel, maxLevel, defaultLevel, libraryVersion, // Constants LEVEL_FASTEST, LEVEL_FAST, LEVEL_DEFAULT, LEVEL_BALANCED, LEVEL_COMPACT, LEVEL_DENSITY, // Error handling errorName, ERROR_MEMORY, ERROR_DST_TOO_SMALL, ERROR_SRC_TOO_SMALL, ERROR_BAD_MAGIC, ERROR_BAD_VERSION, ERROR_BAD_HEADER, ERROR_BAD_CHECKSUM, ERROR_CORRUPT_DATA, ERROR_BAD_OFFSET, ERROR_OVERFLOW, ERROR_IO, ERROR_NULL_INPUT, ERROR_BAD_BLOCK_TYPE, ERROR_BAD_BLOCK_SIZE, }; zxc-0.11.0/wrappers/nodejs/package.json000066400000000000000000000017521520102567100200140ustar00rootroot00000000000000{ "name": "zxc-compress", "version": "0.11.0", "description": "ZXC: High-performance lossless asymmetric compression for Node.js", "main": "lib/index.js", "types": "lib/index.d.ts", "files": [ "lib/", "src/", "CMakeLists.txt" ], "scripts": { "install": "cmake-js compile", "build": "cmake-js compile", "rebuild": "cmake-js rebuild", "test": "vitest run" }, "keywords": [ "compression", "decompression", "lossless", "zxc", "performance", "lz77" ], "author": "Bertrand Lebonnois", "license": "BSD-3-Clause", "repository": { "type": "git", "url": "https://github.com/hellobertrand/zxc.git", "directory": "wrappers/nodejs" }, "homepage": "https://github.com/hellobertrand/zxc", "engines": { "node": ">=16.0.0" }, "dependencies": { "node-addon-api": "^8.5.0", "cmake-js": "^8.0.0" }, "devDependencies": { "vitest": "^4.0.0" }, "binary": { "napi_versions": [ 8 ] } }zxc-0.11.0/wrappers/nodejs/src/000077500000000000000000000000001520102567100163105ustar00rootroot00000000000000zxc-0.11.0/wrappers/nodejs/src/zxc_addon.cc000066400000000000000000000501301520102567100205670ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ #include #include extern "C" { #include "zxc.h" } // ============================================================================= // compressBound(inputSize: number): number // ============================================================================= static Napi::Value CompressBound(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); if (info.Length() < 1 || !info[0].IsNumber()) { Napi::TypeError::New(env, "Expected a number (inputSize)").ThrowAsJavaScriptException(); return env.Undefined(); } uint64_t input_size = info[0].As().Int64Value(); uint64_t bound = zxc_compress_bound(static_cast(input_size)); return Napi::Number::New(env, static_cast(bound)); } // ============================================================================= // compress(buffer: Buffer, level?: number, checksum?: boolean, seekable?: boolean): Buffer // ============================================================================= static Napi::Value Compress(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); if (info.Length() < 1 || !info[0].IsBuffer()) { Napi::TypeError::New(env, "Expected a Buffer as first argument") .ThrowAsJavaScriptException(); return env.Undefined(); } Napi::Buffer src_buf = info[0].As>(); const void* src = src_buf.Data(); size_t src_size = src_buf.Length(); int level = ZXC_LEVEL_DEFAULT; if (info.Length() >= 2 && info[1].IsNumber()) { level = info[1].As().Int32Value(); } int checksum = 0; if (info.Length() >= 3 && info[2].IsBoolean()) { checksum = info[2].As().Value() ? 1 : 0; } int seekable = 0; if (info.Length() >= 4 && info[3].IsBoolean()) { seekable = info[3].As().Value() ? 1 : 0; } // Handle empty input if (src_size == 0 && !checksum) { return Napi::Buffer::New(env, 0); } uint64_t bound = zxc_compress_bound(src_size); Napi::Buffer dst_buf = Napi::Buffer::New(env, static_cast(bound)); zxc_compress_opts_t opts = {0}; opts.level = level; opts.checksum_enabled = checksum; opts.seekable = seekable; int64_t nwritten = zxc_compress(src, src_size, dst_buf.Data(), static_cast(bound), &opts); if (nwritten < 0) { int err_code = static_cast(nwritten); Napi::Error err = Napi::Error::New(env, zxc_error_name(err_code)); err.Set("code", Napi::Number::New(env, err_code)); err.ThrowAsJavaScriptException(); return env.Undefined(); } if (nwritten == 0) { Napi::Error::New(env, "Input is too small to be compressed").ThrowAsJavaScriptException(); return env.Undefined(); } // Return a slice of the buffer with the actual size return Napi::Buffer::Copy(env, dst_buf.Data(), static_cast(nwritten)); } // ============================================================================= // decompress(buffer: Buffer, decompressSize: number, checksum?: boolean): Buffer // ============================================================================= static Napi::Value Decompress(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); if (info.Length() < 2 || !info[0].IsBuffer() || !info[1].IsNumber()) { Napi::TypeError::New(env, "Expected (Buffer, number) as arguments") .ThrowAsJavaScriptException(); return env.Undefined(); } Napi::Buffer src_buf = info[0].As>(); const void* src = src_buf.Data(); size_t src_size = src_buf.Length(); size_t decompress_size = static_cast(info[1].As().Int64Value()); int checksum = 0; if (info.Length() >= 3 && info[2].IsBoolean()) { checksum = info[2].As().Value() ? 1 : 0; } Napi::Buffer dst_buf = Napi::Buffer::New(env, decompress_size); zxc_decompress_opts_t dopts = {0}; dopts.checksum_enabled = checksum; int64_t nwritten = zxc_decompress(src, src_size, dst_buf.Data(), decompress_size, &dopts); if (nwritten < 0) { int err_code = static_cast(nwritten); Napi::Error err = Napi::Error::New(env, zxc_error_name(err_code)); err.Set("code", Napi::Number::New(env, err_code)); err.ThrowAsJavaScriptException(); return env.Undefined(); } return dst_buf; } // ============================================================================= // getDecompressedSize(buffer: Buffer): number // ============================================================================= static Napi::Value GetDecompressedSize(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); if (info.Length() < 1 || !info[0].IsBuffer()) { Napi::TypeError::New(env, "Expected a Buffer as first argument") .ThrowAsJavaScriptException(); return env.Undefined(); } Napi::Buffer src_buf = info[0].As>(); uint64_t size = zxc_get_decompressed_size(src_buf.Data(), src_buf.Length()); return Napi::Number::New(env, static_cast(size)); } // ============================================================================= // minLevel(): number // maxLevel(): number // defaultLevel(): number // libraryVersion(): string // ============================================================================= static Napi::Value MinLevel(const Napi::CallbackInfo& info) { return Napi::Number::New(info.Env(), zxc_min_level()); } static Napi::Value MaxLevel(const Napi::CallbackInfo& info) { return Napi::Number::New(info.Env(), zxc_max_level()); } static Napi::Value DefaultLevel(const Napi::CallbackInfo& info) { return Napi::Number::New(info.Env(), zxc_default_level()); } static Napi::Value LibraryVersion(const Napi::CallbackInfo& info) { const char* v = zxc_version_string(); return Napi::String::New(info.Env(), v ? v : ""); } // ============================================================================= // Push Streaming API (single-threaded, caller-driven) // ============================================================================= // // The C contract drives input/output via mutable structs and reentrant // calls. We expose this to JS as two classes (CStream / DStream) whose // methods take a Buffer in and return a Buffer out, hiding the loop. A // thin Buffer over std::vector growing on demand is used to // accumulate the output across multiple drain rounds. static Napi::Value ThrowZxcError(Napi::Env env, int code) { Napi::Error err = Napi::Error::New(env, zxc_error_name(code)); err.Set("code", Napi::Number::New(env, code)); err.ThrowAsJavaScriptException(); return env.Undefined(); } /* Grow `out` so at least `out_len + want` bytes are addressable. Doubles when * possible, caps at vector::max_size(), and throws if the request can't fit. */ static bool GrowOutput(Napi::Env env, std::vector& out, size_t out_len, size_t want) { const size_t cap = out.max_size(); if (want > cap - out_len) { Napi::Error::New(env, "output buffer size overflow") .ThrowAsJavaScriptException(); return false; } const size_t needed = out_len + want; size_t new_size = (out.size() > cap / 2) ? cap : out.size() * 2; if (new_size < needed) new_size = needed; out.resize(new_size); return true; } class CStreamWrap : public Napi::ObjectWrap { public: static Napi::Function GetClass(Napi::Env env) { return DefineClass(env, "CStream", { InstanceMethod("compress", &CStreamWrap::Compress), InstanceMethod("end", &CStreamWrap::End), InstanceMethod("close", &CStreamWrap::Close), InstanceMethod("inSize", &CStreamWrap::InSize), InstanceMethod("outSize", &CStreamWrap::OutSize), }); } CStreamWrap(const Napi::CallbackInfo& info) : Napi::ObjectWrap(info) { Napi::Env env = info.Env(); zxc_compress_opts_t opts = {0}; opts.level = ZXC_LEVEL_DEFAULT; if (info.Length() >= 1 && info[0].IsObject()) { Napi::Object o = info[0].As(); if (o.Has("level") && o.Get("level").IsNumber()) { opts.level = o.Get("level").As().Int32Value(); } if (o.Has("checksum") && o.Get("checksum").IsBoolean()) { opts.checksum_enabled = o.Get("checksum").As().Value() ? 1 : 0; } if (o.Has("blockSize") && o.Get("blockSize").IsNumber()) { opts.block_size = static_cast(o.Get("blockSize").As().Int64Value()); } } cs_ = zxc_cstream_create(&opts); if (!cs_) { Napi::Error::New(env, "zxc_cstream_create failed").ThrowAsJavaScriptException(); } } ~CStreamWrap() { if (cs_) zxc_cstream_free(cs_); } private: zxc_cstream* cs_ = nullptr; bool requireOpen(Napi::Env env) { if (!cs_) { Napi::Error::New(env, "CStream is closed").ThrowAsJavaScriptException(); return false; } return true; } /* Drains from `cs` until either a stop condition is reached. The * caller-supplied `step` decides whether one iteration of * zxc_cstream_compress / _end has fully drained. */ Napi::Value DrainCompress(Napi::Env env, const uint8_t* src, size_t srcLen) { std::vector out; size_t cap = zxc_cstream_out_size(cs_); if (cap < 4096) cap = 4096; out.resize(cap); size_t out_len = 0; zxc_inbuf_t in = {src, srcLen, 0}; for (;;) { size_t want = zxc_cstream_out_size(cs_); if (want < 4096) want = 4096; if (out.size() - out_len < want) { if (!GrowOutput(env, out, out_len, want)) return env.Undefined(); } zxc_outbuf_t obuf = {out.data() + out_len, out.size() - out_len, 0}; int64_t r = zxc_cstream_compress(cs_, &obuf, &in); out_len += obuf.pos; if (r < 0) return ThrowZxcError(env, static_cast(r)); if (r == 0 && in.pos == in.size) break; } return Napi::Buffer::Copy(env, out.data(), out_len); } Napi::Value DrainEnd(Napi::Env env) { std::vector out; size_t cap = zxc_cstream_out_size(cs_); if (cap < 4096) cap = 4096; out.resize(cap); size_t out_len = 0; for (;;) { if (out.size() - out_len < 4096) { if (!GrowOutput(env, out, out_len, 4096)) return env.Undefined(); } zxc_outbuf_t obuf = {out.data() + out_len, out.size() - out_len, 0}; int64_t r = zxc_cstream_end(cs_, &obuf); out_len += obuf.pos; if (r < 0) return ThrowZxcError(env, static_cast(r)); if (r == 0) break; } return Napi::Buffer::Copy(env, out.data(), out_len); } Napi::Value Compress(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); if (!requireOpen(env)) return env.Undefined(); if (info.Length() < 1 || !info[0].IsBuffer()) { Napi::TypeError::New(env, "Expected a Buffer").ThrowAsJavaScriptException(); return env.Undefined(); } Napi::Buffer buf = info[0].As>(); return DrainCompress(env, buf.Data(), buf.Length()); } Napi::Value End(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); if (!requireOpen(env)) return env.Undefined(); return DrainEnd(env); } Napi::Value Close(const Napi::CallbackInfo& info) { if (cs_) { zxc_cstream_free(cs_); cs_ = nullptr; } return info.Env().Undefined(); } Napi::Value InSize(const Napi::CallbackInfo& info) { return Napi::Number::New(info.Env(), static_cast(cs_ ? zxc_cstream_in_size(cs_) : 0)); } Napi::Value OutSize(const Napi::CallbackInfo& info) { return Napi::Number::New(info.Env(), static_cast(cs_ ? zxc_cstream_out_size(cs_) : 0)); } }; class DStreamWrap : public Napi::ObjectWrap { public: static Napi::Function GetClass(Napi::Env env) { return DefineClass(env, "DStream", { InstanceMethod("decompress", &DStreamWrap::Decompress), InstanceMethod("close", &DStreamWrap::Close), InstanceMethod("finished", &DStreamWrap::Finished), InstanceMethod("inSize", &DStreamWrap::InSize), InstanceMethod("outSize", &DStreamWrap::OutSize), }); } DStreamWrap(const Napi::CallbackInfo& info) : Napi::ObjectWrap(info) { Napi::Env env = info.Env(); zxc_decompress_opts_t opts = {0}; if (info.Length() >= 1 && info[0].IsObject()) { Napi::Object o = info[0].As(); if (o.Has("checksum") && o.Get("checksum").IsBoolean()) { opts.checksum_enabled = o.Get("checksum").As().Value() ? 1 : 0; } } ds_ = zxc_dstream_create(&opts); if (!ds_) { Napi::Error::New(env, "zxc_dstream_create failed").ThrowAsJavaScriptException(); } } ~DStreamWrap() { if (ds_) zxc_dstream_free(ds_); } private: zxc_dstream* ds_ = nullptr; bool requireOpen(Napi::Env env) { if (!ds_) { Napi::Error::New(env, "DStream is closed").ThrowAsJavaScriptException(); return false; } return true; } Napi::Value Decompress(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); if (!requireOpen(env)) return env.Undefined(); if (info.Length() < 1 || !info[0].IsBuffer()) { Napi::TypeError::New(env, "Expected a Buffer").ThrowAsJavaScriptException(); return env.Undefined(); } Napi::Buffer buf = info[0].As>(); std::vector out; size_t cap = zxc_dstream_out_size(ds_); if (cap < 4096) cap = 4096; out.resize(cap); size_t out_len = 0; zxc_inbuf_t in = {buf.Data(), buf.Length(), 0}; for (;;) { size_t want = zxc_dstream_out_size(ds_); if (want < 4096) want = 4096; if (out.size() - out_len < want) { if (!GrowOutput(env, out, out_len, want)) return env.Undefined(); } zxc_outbuf_t obuf = {out.data() + out_len, out.size() - out_len, 0}; zxc_inbuf_t empty_in = {nullptr, 0, 0}; zxc_inbuf_t* cur_in = (in.pos < in.size) ? &in : &empty_in; const size_t before_in = cur_in->pos; const size_t before_out = obuf.pos; const int64_t r = zxc_dstream_decompress(ds_, &obuf, cur_in); out_len += obuf.pos; if (r < 0) return ThrowZxcError(env, static_cast(r)); /* Keep draining even after input is exhausted; stop only when * no progress was made (no input consumed AND no output produced). */ if (cur_in->pos == before_in && obuf.pos == before_out) break; } return Napi::Buffer::Copy(env, out.data(), out_len); } Napi::Value Finished(const Napi::CallbackInfo& info) { return Napi::Boolean::New(info.Env(), ds_ ? zxc_dstream_finished(ds_) != 0 : false); } Napi::Value Close(const Napi::CallbackInfo& info) { if (ds_) { zxc_dstream_free(ds_); ds_ = nullptr; } return info.Env().Undefined(); } Napi::Value InSize(const Napi::CallbackInfo& info) { return Napi::Number::New(info.Env(), static_cast(ds_ ? zxc_dstream_in_size(ds_) : 0)); } Napi::Value OutSize(const Napi::CallbackInfo& info) { return Napi::Number::New(info.Env(), static_cast(ds_ ? zxc_dstream_out_size(ds_) : 0)); } }; // ============================================================================= // errorName(code: number): string // ============================================================================= static Napi::Value ErrorName(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); if (info.Length() < 1 || !info[0].IsNumber()) { Napi::TypeError::New(env, "Expected a number (error code)").ThrowAsJavaScriptException(); return env.Undefined(); } int code = info[0].As().Int32Value(); const char* name = zxc_error_name(code); return Napi::String::New(env, name); } // ============================================================================= // Module initialization // ============================================================================= static Napi::Object Init(Napi::Env env, Napi::Object exports) { // Functions exports.Set("compressBound", Napi::Function::New(env, CompressBound, "compressBound")); exports.Set("compress", Napi::Function::New(env, Compress, "compress")); exports.Set("decompress", Napi::Function::New(env, Decompress, "decompress")); exports.Set("getDecompressedSize", Napi::Function::New(env, GetDecompressedSize, "getDecompressedSize")); exports.Set("errorName", Napi::Function::New(env, ErrorName, "errorName")); // Library info helpers exports.Set("minLevel", Napi::Function::New(env, MinLevel, "minLevel")); exports.Set("maxLevel", Napi::Function::New(env, MaxLevel, "maxLevel")); exports.Set("defaultLevel", Napi::Function::New(env, DefaultLevel, "defaultLevel")); exports.Set("libraryVersion", Napi::Function::New(env, LibraryVersion, "libraryVersion")); // Push streaming classes exports.Set("CStream", CStreamWrap::GetClass(env)); exports.Set("DStream", DStreamWrap::GetClass(env)); // Compression level constants exports.Set("LEVEL_FASTEST", Napi::Number::New(env, ZXC_LEVEL_FASTEST)); exports.Set("LEVEL_FAST", Napi::Number::New(env, ZXC_LEVEL_FAST)); exports.Set("LEVEL_DEFAULT", Napi::Number::New(env, ZXC_LEVEL_DEFAULT)); exports.Set("LEVEL_BALANCED", Napi::Number::New(env, ZXC_LEVEL_BALANCED)); exports.Set("LEVEL_COMPACT", Napi::Number::New(env, ZXC_LEVEL_COMPACT)); exports.Set("LEVEL_DENSITY", Napi::Number::New(env, ZXC_LEVEL_DENSITY)); // Error constants exports.Set("ERROR_MEMORY", Napi::Number::New(env, ZXC_ERROR_MEMORY)); exports.Set("ERROR_DST_TOO_SMALL", Napi::Number::New(env, ZXC_ERROR_DST_TOO_SMALL)); exports.Set("ERROR_SRC_TOO_SMALL", Napi::Number::New(env, ZXC_ERROR_SRC_TOO_SMALL)); exports.Set("ERROR_BAD_MAGIC", Napi::Number::New(env, ZXC_ERROR_BAD_MAGIC)); exports.Set("ERROR_BAD_VERSION", Napi::Number::New(env, ZXC_ERROR_BAD_VERSION)); exports.Set("ERROR_BAD_HEADER", Napi::Number::New(env, ZXC_ERROR_BAD_HEADER)); exports.Set("ERROR_BAD_CHECKSUM", Napi::Number::New(env, ZXC_ERROR_BAD_CHECKSUM)); exports.Set("ERROR_CORRUPT_DATA", Napi::Number::New(env, ZXC_ERROR_CORRUPT_DATA)); exports.Set("ERROR_BAD_OFFSET", Napi::Number::New(env, ZXC_ERROR_BAD_OFFSET)); exports.Set("ERROR_OVERFLOW", Napi::Number::New(env, ZXC_ERROR_OVERFLOW)); exports.Set("ERROR_IO", Napi::Number::New(env, ZXC_ERROR_IO)); exports.Set("ERROR_NULL_INPUT", Napi::Number::New(env, ZXC_ERROR_NULL_INPUT)); exports.Set("ERROR_BAD_BLOCK_TYPE", Napi::Number::New(env, ZXC_ERROR_BAD_BLOCK_TYPE)); exports.Set("ERROR_BAD_BLOCK_SIZE", Napi::Number::New(env, ZXC_ERROR_BAD_BLOCK_SIZE)); return exports; } NODE_API_MODULE(zxc_nodejs, Init) zxc-0.11.0/wrappers/nodejs/test/000077500000000000000000000000001520102567100165005ustar00rootroot00000000000000zxc-0.11.0/wrappers/nodejs/test/zxc.streams.test.js000066400000000000000000000111141520102567100222730ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause * * Tests for the stream.Transform adapters and detectZxc helper. */ 'use strict'; const { Readable, PassThrough } = require('node:stream'); const { pipeline } = require('node:stream/promises'); const zxc = require('../lib/index'); async function gather(stream) { const chunks = []; for await (const c of stream) chunks.push(c); return Buffer.concat(chunks); } async function roundtripPipeline(data, opts = {}) { const compressed = await gather( Readable.from([data]).pipe(zxc.createCompressStream(opts)), ); const decompressed = await gather( Readable.from([compressed]).pipe(zxc.createDecompressStream(opts)), ); return decompressed; } describe('CompressStream / DecompressStream roundtrip', () => { test('small buffer', async () => { const data = Buffer.from('Hello stream.Transform bridge over ZXC!'); const got = await roundtripPipeline(data); expect(Buffer.compare(got, data)).toBe(0); }); test('large 2 MB buffer', async () => { const data = Buffer.alloc(2 * 1024 * 1024); for (let i = 0; i < data.length; i++) data[i] = (i * 13) % 251; const got = await roundtripPipeline(data); expect(got.length).toBe(data.length); expect(Buffer.compare(got, data)).toBe(0); }); test('many small chunks fed sequentially', async () => { const want = Buffer.alloc(64 * 1024); for (let i = 0; i < want.length; i++) want[i] = (i ^ 0x5A) & 0xFF; // Feed 257-byte chunks to exercise buffering boundaries. const chunks = []; for (let i = 0; i < want.length; i += 257) { chunks.push(want.subarray(i, Math.min(i + 257, want.length))); } const compressed = await gather( Readable.from(chunks).pipe(zxc.createCompressStream()), ); const got = await gather( Readable.from([compressed]).pipe(zxc.createDecompressStream()), ); expect(Buffer.compare(got, want)).toBe(0); }); test('with checksum option', async () => { const data = Buffer.alloc(32 * 1024); for (let i = 0; i < data.length; i++) data[i] = i & 0xFF; const got = await roundtripPipeline(data, { checksum: true }); expect(Buffer.compare(got, data)).toBe(0); }); test('via pipeline()', async () => { const data = Buffer.from('pipeline integration smoke test'.repeat(100)); const intermediate = new PassThrough(); const finalSink = new PassThrough(); const collected = gather(finalSink); await pipeline( Readable.from([data]), zxc.createCompressStream(), intermediate, zxc.createDecompressStream(), finalSink, ); expect(Buffer.compare(await collected, data)).toBe(0); }); }); describe('DecompressStream error paths', () => { test('truncated frame emits ZXC_TRUNCATED', async () => { const data = Buffer.alloc(32 * 1024, 0x41); const compressed = await gather( Readable.from([data]).pipe(zxc.createCompressStream()), ); const truncated = compressed.subarray(0, Math.floor(compressed.length / 2)); await expect( gather(Readable.from([truncated]).pipe(zxc.createDecompressStream())), ).rejects.toMatchObject({ code: 'ZXC_TRUNCATED' }); }); }); describe('detectZxc', () => { test('detects a frame produced by compress()', () => { const frame = zxc.compress(Buffer.from('sniff me')); expect(zxc.detectZxc(frame)).toBe(true); }); test('detects a frame produced by CompressStream', async () => { const frame = await gather( Readable.from([Buffer.from('hi')]).pipe(zxc.createCompressStream()), ); expect(zxc.detectZxc(frame)).toBe(true); }); test.each([ ['empty', Buffer.alloc(0)], ['too short', Buffer.from([0xF5, 0x2E, 0xB0])], ['zeros', Buffer.alloc(4)], ['random text', Buffer.from('not a zxc frame at all')], ])('rejects %s', (_name, buf) => { expect(zxc.detectZxc(buf)).toBe(false); }); test('accepts Uint8Array', () => { const frame = zxc.compress(Buffer.from('x')); const u8 = new Uint8Array(frame); expect(zxc.detectZxc(u8)).toBe(true); }); test('returns false for null/undefined', () => { expect(zxc.detectZxc(null)).toBe(false); expect(zxc.detectZxc(undefined)).toBe(false); }); }); zxc-0.11.0/wrappers/nodejs/test/zxc.test.js000066400000000000000000000223001520102567100206150ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ 'use strict'; const zxc = require('../lib/index'); // ============================================================================= // Roundtrip tests // ============================================================================= describe('compress/decompress roundtrip', () => { const testCases = [ { name: 'normal data', data: Buffer.from('hello world'.repeat(10)) }, { name: 'single byte', data: Buffer.from('a') }, { name: 'empty buffer', data: Buffer.alloc(0) }, { name: 'large 10 MB', data: Buffer.alloc(10_000_000, 0x42) }, ]; for (const { name, data } of testCases) { test(`roundtrip: ${name}`, () => { for (let level = zxc.LEVEL_FASTEST; level <= zxc.LEVEL_DENSITY; level++) { const compressed = zxc.compress(data, { level }); const size = zxc.getDecompressedSize(compressed); const decompressed = zxc.decompress(compressed, { size }); expect(decompressed.length).toBe(data.length); expect(Buffer.compare(decompressed, data)).toBe(0); } }); } test('roundtrip with auto-size detection', () => { const data = Buffer.from('auto size detection test'.repeat(100)); const compressed = zxc.compress(data); const decompressed = zxc.decompress(compressed); expect(decompressed.length).toBe(data.length); expect(Buffer.compare(decompressed, data)).toBe(0); }); }); // ============================================================================= // compressBound // ============================================================================= describe('compressBound', () => { test('returns a value >= input size', () => { const bound = zxc.compressBound(1024); expect(bound).toBeGreaterThanOrEqual(1024); }); test('returns 0-based bound for size 0', () => { const bound = zxc.compressBound(0); expect(bound).toBeGreaterThanOrEqual(0); }); }); // ============================================================================= // Corruption detection // ============================================================================= describe('corruption detection', () => { test('detects corrupted data with checksum', () => { const data = Buffer.from('hello world'.repeat(10)); const compressed = zxc.compress(data, { checksum: true }); // Corrupt last byte const corrupted = Buffer.from(compressed); corrupted[corrupted.length - 1] ^= 0x01; expect(() => { zxc.decompress(corrupted, { size: data.length, checksum: true }); }).toThrow(); }); test('throws on truncated compressed input', () => { expect(() => { zxc.decompress(Buffer.from([0x01, 0x02, 0x03]), { size: 100 }); }).toThrow(); }); }); // ============================================================================= // Invalid input types // ============================================================================= describe('invalid input handling', () => { test('compress rejects non-Buffer', () => { expect(() => zxc.compress('string')).toThrow(TypeError); expect(() => zxc.compress(123)).toThrow(TypeError); expect(() => zxc.compress(null)).toThrow(TypeError); }); test('decompress rejects non-Buffer', () => { expect(() => zxc.decompress('string')).toThrow(TypeError); expect(() => zxc.decompress(123)).toThrow(TypeError); }); test('getDecompressedSize rejects non-Buffer', () => { expect(() => zxc.getDecompressedSize('string')).toThrow(TypeError); }); test('compressBound rejects non-number', () => { expect(() => zxc.compressBound('abc')).toThrow(TypeError); }); }); // ============================================================================= // Constants // ============================================================================= describe('constants', () => { test('compression levels are defined', () => { expect(zxc.LEVEL_FASTEST).toBe(1); expect(zxc.LEVEL_FAST).toBe(2); expect(zxc.LEVEL_DEFAULT).toBe(3); expect(zxc.LEVEL_BALANCED).toBe(4); expect(zxc.LEVEL_COMPACT).toBe(5); expect(zxc.LEVEL_DENSITY).toBe(6); }); }); // ============================================================================= // Error Handling // ============================================================================= describe('error handling', () => { test('error constants are defined', () => { expect(zxc.ERROR_MEMORY).toBe(-1); expect(zxc.ERROR_DST_TOO_SMALL).toBe(-2); expect(zxc.ERROR_SRC_TOO_SMALL).toBe(-3); expect(zxc.ERROR_BAD_MAGIC).toBe(-4); expect(zxc.ERROR_BAD_VERSION).toBe(-5); expect(zxc.ERROR_BAD_HEADER).toBe(-6); expect(zxc.ERROR_BAD_CHECKSUM).toBe(-7); expect(zxc.ERROR_CORRUPT_DATA).toBe(-8); expect(zxc.ERROR_BAD_OFFSET).toBe(-9); expect(zxc.ERROR_OVERFLOW).toBe(-10); expect(zxc.ERROR_IO).toBe(-11); expect(zxc.ERROR_NULL_INPUT).toBe(-12); expect(zxc.ERROR_BAD_BLOCK_TYPE).toBe(-13); expect(zxc.ERROR_BAD_BLOCK_SIZE).toBe(-14); }); test('errorName works', () => { expect(zxc.errorName(zxc.ERROR_DST_TOO_SMALL)).toBe('ZXC_ERROR_DST_TOO_SMALL'); expect(zxc.errorName(-999)).toBe('ZXC_UNKNOWN_ERROR'); }); test('thrown errors have code property', () => { try { zxc.decompress(Buffer.from([0x01, 0x02, 0x03]), { size: 100 }); throw new Error('Should have thrown'); } catch (err) { expect(err.code).toBe(zxc.ERROR_NULL_INPUT); expect(err.message).toBe('ZXC_ERROR_NULL_INPUT'); } }); test('detects bad magic with correct code', () => { try { // Provide enough bytes (16 for header) but fake magic const fakeHeader = Buffer.alloc(32); fakeHeader.write('FAKE', 0, 'utf8'); zxc.decompress(fakeHeader, { size: 100 }); throw new Error('Should have thrown'); } catch (err) { expect(err.code).toBe(zxc.ERROR_BAD_HEADER); } }); }); // ============================================================================= // Push Streaming API (zxc.CStream / zxc.DStream) // ============================================================================= function pstreamRoundtrip(data, { level, checksum } = {}) { const cs = new zxc.CStream({ level, checksum }); const compressedChunks = []; // Feed in 17-byte slices to exercise the buffering/state machine. const step = Math.max(1, Math.min(17, data.length)); for (let i = 0; i < data.length; i += step) { compressedChunks.push(cs.compress(data.subarray(i, i + step))); } compressedChunks.push(cs.end()); cs.close(); const compressed = Buffer.concat(compressedChunks); const ds = new zxc.DStream({ checksum }); const outChunks = []; const dstep = Math.max(1, Math.min(31, compressed.length)); for (let i = 0; i < compressed.length; i += dstep) { outChunks.push(ds.decompress(compressed.subarray(i, i + dstep))); } expect(ds.finished()).toBe(true); ds.close(); return Buffer.concat(outChunks); } describe('pstream roundtrip', () => { test('small payload', () => { const data = Buffer.from('Hello pstream! Round-trip through the Node.js push API.'); expect(pstreamRoundtrip(data).equals(data)).toBe(true); }); test('with checksum', () => { const data = Buffer.alloc(32 * 1024); for (let i = 0; i < data.length; i++) data[i] = i % 251; expect(pstreamRoundtrip(data, { checksum: true }).equals(data)).toBe(true); }); test('multi-block (>512 KB)', () => { const data = Buffer.alloc(1536 * 1024); for (let i = 0; i < data.length; i++) data[i] = (i * 7) % 256; expect(pstreamRoundtrip(data).equals(data)).toBe(true); }); }); describe('pstream lifecycle', () => { test('size hints non-zero', () => { const cs = new zxc.CStream(); expect(cs.inSize()).toBeGreaterThan(0); expect(cs.outSize()).toBeGreaterThan(0); cs.close(); const ds = new zxc.DStream(); expect(ds.inSize()).toBeGreaterThan(0); expect(ds.outSize()).toBeGreaterThan(0); expect(ds.finished()).toBe(false); ds.close(); }); test('use after close throws', () => { const cs = new zxc.CStream(); cs.close(); expect(() => cs.compress(Buffer.from('x'))).toThrow(/closed/); const ds = new zxc.DStream(); ds.close(); expect(() => ds.decompress(Buffer.from('x'))).toThrow(/closed/); }); test('truncated stream not finished', () => { const cs = new zxc.CStream(); const compressed = Buffer.concat([cs.compress(Buffer.from('hello '.repeat(1000))), cs.end()]); cs.close(); const ds = new zxc.DStream(); ds.decompress(compressed.subarray(0, compressed.length - 5)); expect(ds.finished()).toBe(false); ds.close(); }); }); zxc-0.11.0/wrappers/nodejs/vitest.config.mjs000066400000000000000000000001631520102567100210160ustar00rootroot00000000000000import { defineConfig } from 'vitest/config'; export default defineConfig({ test: { globals: true, }, }); zxc-0.11.0/wrappers/python/000077500000000000000000000000001520102567100155605ustar00rootroot00000000000000zxc-0.11.0/wrappers/python/.gitignore000066400000000000000000000014011520102567100175440ustar00rootroot00000000000000# Editor temporary/working/backup files # ######################################### .#* [#]*# *~ *$ *.bak *.diff .idea/ *.iml *.ipr *.iws *.org .project pmip *.rej .settings/ .*.sw[nop] .sw[nop] *.tmp *.vim .vscode tags cscope.out # gnu global GPATH GRTAGS GSYMS GTAGS .cache .mypy_cache/ # Compiled source # ################### *.a *.com *.class *.dll *.exe *.o *.o.d *.py[ocd] *.so *.mod # Compiled source # ################### build build-* _build _version.py # Egg metadata *.egg-info venv/ # OS generated files # ###################### .DS_Store* .VolumeIcon.icns .fseventsd Icon? .gdb_history ehthumbs.db Thumbs.db .directory # pytest generated files # ########################## /.pytest_cache # benchmarking # ########################## heaptrack* *.out zxc-0.11.0/wrappers/python/CMakeLists.txt000066400000000000000000000010751520102567100203230ustar00rootroot00000000000000cmake_minimum_required(VERSION 3.20) project(zxc_python C) find_package(Python COMPONENTS Development.Module REQUIRED) # ---- Core ---- add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../../ ${CMAKE_CURRENT_BINARY_DIR}/zxc_core_build) # ---- Python extension ---- Python_add_library(_zxc MODULE src/zxc/_zxc.c) # Without "lib" prefix set_target_properties(_zxc PROPERTIES PREFIX "") target_include_directories(_zxc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../include ) target_link_libraries(_zxc PRIVATE zxc_lib) install(TARGETS _zxc DESTINATION zxc)zxc-0.11.0/wrappers/python/README.md000066400000000000000000000013751520102567100170450ustar00rootroot00000000000000# ZXC Python Bindings High-performance Python bindings for the **ZXC** asymmetric compressor, optimized for **fast decompression**. Designed for *Write Once, Read Many* workloads like ML datasets, game assets, and caches. ## Features - **Blazing fast decompression** - ZXC is specifically optimized for read-heavy workloads. - **Buffer protocol support** - works with `bytes`, `bytearray`, `memoryview`, and even NumPy arrays. - **Releases the GIL** during compression/decompression - true parallelism with Python threads. - **Stream helpers** - compress/decompress file-like objects. ## Installation (from source) ```bash git clone https://github.com/hellobertrand/zxc.git cd zxc/wrappers/python python -m venv .venv source .venv/bin/activate pip install .zxc-0.11.0/wrappers/python/pyproject.toml000066400000000000000000000011341520102567100204730ustar00rootroot00000000000000[build-system] requires = ["scikit-build-core>=0.9", "setuptools_scm[toml]>=7.0"] build-backend = "scikit_build_core.build" [project] name = "zxc-compress" dynamic = ["version"] description = "ZXC: Package for high-performance, lossless, asymmetric compressions" readme = "README.md" license = "BSD-3-Clause" [project.urls] Homepage = "https://github.com/hellobertrand/zxc" [tool.scikit-build] metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" wheel.packages = ["src/zxc"] [tool.setuptools_scm] root = "../../" version_scheme = "only-version" local_scheme = "no-local-version"zxc-0.11.0/wrappers/python/src/000077500000000000000000000000001520102567100163475ustar00rootroot00000000000000zxc-0.11.0/wrappers/python/src/zxc/000077500000000000000000000000001520102567100171535ustar00rootroot00000000000000zxc-0.11.0/wrappers/python/src/zxc/__init__.py000066400000000000000000000424201520102567100212660ustar00rootroot00000000000000""" ZXC - High-performance lossless compression Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. SPDX-License-Identifier: BSD-3-Clause """ import io as _io from ._zxc import ( pyzxc_compress, pyzxc_decompress, pyzxc_stream_compress, pyzxc_stream_decompress, pyzxc_get_decompressed_size, pyzxc_min_level, pyzxc_max_level, pyzxc_default_level, pyzxc_version_string, pyzxc_cstream_create, pyzxc_cstream_compress, pyzxc_cstream_end, pyzxc_cstream_in_size, pyzxc_cstream_out_size, pyzxc_cstream_free, pyzxc_dstream_create, pyzxc_dstream_decompress, pyzxc_dstream_finished, pyzxc_dstream_in_size, pyzxc_dstream_out_size, pyzxc_dstream_free, LEVEL_FASTEST, LEVEL_FAST, LEVEL_DEFAULT, LEVEL_BALANCED, LEVEL_COMPACT, LEVEL_DENSITY, ERROR_MEMORY, ERROR_DST_TOO_SMALL, ERROR_SRC_TOO_SMALL, ERROR_BAD_MAGIC, ERROR_BAD_VERSION, ERROR_BAD_HEADER, ERROR_BAD_CHECKSUM, ERROR_CORRUPT_DATA, ERROR_BAD_OFFSET, ERROR_OVERFLOW, ERROR_IO, ERROR_NULL_INPUT, ERROR_BAD_BLOCK_TYPE, ERROR_BAD_BLOCK_SIZE, ) try: from ._version import __version__ except ImportError: __version__ = "0.0.0-dev" __all__ = [ # Functions "compress", "decompress", "stream_compress", "stream_decompress", "get_decompressed_size", # Push streaming "CStream", "DStream", # io.RawIOBase adapters "ZxcReader", "ZxcWriter", "detect_zxc", # Library info helpers "min_level", "max_level", "default_level", "library_version", # Constants "LEVEL_FASTEST", "LEVEL_FAST", "LEVEL_DEFAULT", "LEVEL_BALANCED", "LEVEL_COMPACT", "LEVEL_DENSITY", # Error Constants "ERROR_MEMORY", "ERROR_DST_TOO_SMALL", "ERROR_SRC_TOO_SMALL", "ERROR_BAD_MAGIC", "ERROR_BAD_VERSION", "ERROR_BAD_HEADER", "ERROR_BAD_CHECKSUM", "ERROR_CORRUPT_DATA", "ERROR_BAD_OFFSET", "ERROR_OVERFLOW", "ERROR_IO", "ERROR_NULL_INPUT", "ERROR_BAD_BLOCK_TYPE", "ERROR_BAD_BLOCK_SIZE", ] def min_level() -> int: """Return the minimum supported compression level (currently 1).""" return pyzxc_min_level() def max_level() -> int: """Return the maximum supported compression level (currently 5).""" return pyzxc_max_level() def default_level() -> int: """Return the default compression level (currently 3).""" return pyzxc_default_level() def library_version() -> str: """Return the version string reported by the linked native libzxc (e.g. ``"0.11.0"``). Distinct from the Python package ``__version__``.""" return pyzxc_version_string() def compress(data, level = LEVEL_DEFAULT, checksum = False) -> bytes: """Compress a bytes object. Args: data: Bytes-like object to compress. level: Compression level. Use constants like LEVEL_FASTEST, LEVEL_DEFAULT, etc. checksum: If True, append a checksum for integrity verification. Returns: Compressed bytes. Note: This function operates entirely in-memory. For streaming files, use `stream_compress`. """ if len(data) == 0 and not checksum: return data return pyzxc_compress(data, level, checksum) def get_decompressed_size(data: bytes) -> int: """Get the original decompressed size of a ZXC compressed buffer. Args: data (bytes): Compressed bytes buffer. Returns: int: Original uncompressed size in bytes, or 0 if the buffer is invalid or too small. Note: This function does not decompress the data, it only reads the footer for size info. """ return pyzxc_get_decompressed_size(data) def decompress(data, decompress_size=None, checksum=False) -> bytes: """Decompress a bytes object. Args: data: Compressed bytes. decompress_size: Expected size. If None, read from header (slower/safer). checksum: If True, verify the checksum appended during compression. Returns: Decompressed bytes. """ if len(data) == 0 and not checksum: return data if decompress_size is None: decompress_size = get_decompressed_size(data) if decompress_size == 0: raise ValueError( "Invalid ZXC header or data too short to determine size" ) return pyzxc_decompress(data, decompress_size, checksum) def stream_compress(src, dst, n_threads=0, level=LEVEL_DEFAULT, checksum=False, seekable=False) -> int: """Compress data from src to dst (file-like objects). Args: src: Readable file-like object with `fileno()` support (e.g., open file). dst: Writable file-like object with `fileno()` support. n_threads: Number of threads to use for compression. 0 uses default. level: Compression level. Use constants like LEVEL_FASTEST, LEVEL_DEFAULT, etc. checksum: If True, append a checksum for integrity verification. seekable: If True, append a seek table for random-access decompression. Returns: Number of bytes written to `dst`. Note: In-memory streams like `io.BytesIO` are not supported. Use the in-memory `compress`/`decompress` functions for buffers. """ if not hasattr(src, "fileno") or not hasattr(dst, "fileno"): raise ValueError("src and dst must be open file-like objects") if not src.readable(): raise ValueError("Source file must be readable") if not dst.writable(): raise ValueError("Destination file must be writable") # CRITICAL: Flush Python buffers before passing FDs to C # to prevent data reordering/corruption. if hasattr(src, "flush"): src.flush() if hasattr(dst, "flush"): dst.flush() return pyzxc_stream_compress(src, dst, n_threads, level, checksum, seekable) def stream_decompress(src, dst, n_threads=0, checksum=False) -> int: """Decompress data from src to dst (file-like objects). Args: src: Readable file-like object with `fileno()` support. dst: Writable file-like object with `fileno()` support. n_threads: Number of threads to use for decompression. 0 uses default. checksum: If True, verify the checksum appended during compression. Returns: Number of bytes written to `dst`. Note: In-memory streams like `io.BytesIO` are not supported. Use the in-memory `compress`/`decompress` functions for buffers. """ if not hasattr(src, "fileno") or not hasattr(dst, "fileno"): raise ValueError("src and dst must be open file-like objects") if not src.readable(): raise ValueError("Source file must be readable") if not dst.writable(): raise ValueError("Destination file must be writable") # CRITICAL: Flush Python buffers before passing FDs to C # to prevent data reordering/corruption. if hasattr(src, "flush"): src.flush() if hasattr(dst, "flush"): dst.flush() return pyzxc_stream_decompress(src, dst, n_threads, checksum) class CStream: """Push-based, single-threaded compression stream. The Python counterpart of the C ``zxc_cstream``. Use this when the multi-threaded ``stream_compress`` (which takes ``FILE*``) is not appropriate - e.g. async event loops, in-memory streams (``io.BytesIO``), network protocols, or callback-driven libraries. The stream is not thread-safe. Each call to :meth:`compress` returns the compressed bytes produced from the input fed in this call (may be empty if the input was small enough to stay inside the internal block accumulator). :meth:`end` flushes the residual block, the EOF marker and the file footer; you MUST call it to produce a valid ZXC archive. Example:: cs = zxc.CStream(level=zxc.LEVEL_DEFAULT, checksum=True) chunks = [cs.compress(part) for part in source] chunks.append(cs.end()) archive = b"".join(chunks) cs.close() Supports the context-manager protocol:: with zxc.CStream(level=3) as cs: out = cs.compress(data) + cs.end() """ __slots__ = ("_handle",) def __init__(self, level: int = LEVEL_DEFAULT, checksum: bool = False, block_size: int = 0): self._handle = pyzxc_cstream_create(level, checksum, block_size) def compress(self, data) -> bytes: """Push *data* into the stream and return any compressed bytes produced this call. May return ``b""`` if the input fit entirely into the internal block accumulator. The returned bytes must be written verbatim to the sink in order. """ if self._handle is None: raise ValueError("CStream is closed") return pyzxc_cstream_compress(self._handle, data) def end(self) -> bytes: """Finalise the stream and return the trailing bytes (residual block + EOF + file footer). Call exactly once after the last :meth:`compress`. """ if self._handle is None: raise ValueError("CStream is closed") return pyzxc_cstream_end(self._handle) @property def in_size(self) -> int: """Suggested input chunk size (block size, in bytes).""" if self._handle is None: return 0 return pyzxc_cstream_in_size(self._handle) @property def out_size(self) -> int: """Suggested output chunk size to never trigger a partial drain.""" if self._handle is None: return 0 return pyzxc_cstream_out_size(self._handle) def close(self) -> None: """Release native resources. Idempotent.""" if self._handle is not None: pyzxc_cstream_free(self._handle) self._handle = None def __enter__(self): return self def __exit__(self, exc_type, exc, tb): self.close() return False def __del__(self): try: self.close() except Exception: pass class DStream: """Push-based, single-threaded decompression stream. The Python counterpart of the C ``zxc_dstream``. Feed compressed bytes via :meth:`decompress`; each call returns the decompressed bytes produced so far. After all input has been fed, :attr:`finished` becomes ``True`` once the file footer is validated. Example:: ds = zxc.DStream(checksum=True) out = b"".join(ds.decompress(chunk) for chunk in compressed_chunks) if not ds.finished: raise ValueError("truncated stream") ds.close() """ __slots__ = ("_handle",) def __init__(self, checksum: bool = False): self._handle = pyzxc_dstream_create(checksum) def decompress(self, data) -> bytes: """Push *data* and return the decompressed bytes produced. May return ``b""`` if the parser is still waiting for more input (e.g. mid-header). """ if self._handle is None: raise ValueError("DStream is closed") return pyzxc_dstream_decompress(self._handle, data) @property def finished(self) -> bool: """``True`` once the decoder has reached and validated the file footer. Useful to detect truncated input.""" if self._handle is None: return False return pyzxc_dstream_finished(self._handle) @property def in_size(self) -> int: """Suggested input chunk size for the decompressor.""" if self._handle is None: return 0 return pyzxc_dstream_in_size(self._handle) @property def out_size(self) -> int: """Suggested output chunk size for the decompressor.""" if self._handle is None: return 0 return pyzxc_dstream_out_size(self._handle) def close(self) -> None: """Release native resources. Idempotent.""" if self._handle is not None: pyzxc_dstream_free(self._handle) self._handle = None def __enter__(self): return self def __exit__(self, exc_type, exc, tb): self.close() return False def __del__(self): try: self.close() except Exception: pass # ============================================================================ # io.RawIOBase adapters over the push streaming API # ============================================================================ # # Mirror of the wrappers shipped by the Go and Rust bindings: turn a # CStream / DStream pair into Python's standard binary file-like protocol so # ZXC can be plugged into pipelines that expect it — tarfile, requests, # oras-py / OCI registry clients, etc. # # Wrap with ``io.BufferedReader`` / ``io.BufferedWriter`` for buffering when # performance matters. class ZxcReader(_io.RawIOBase): """Decompresses a ZXC frame read from a binary file-like object. Implements the standard :class:`io.RawIOBase` interface so it can be plugged into any code that expects a readable binary stream. Raises :class:`OSError` with ``errno=None`` if the underlying source is drained before the ZXC footer is reached (truncated frame). Example:: with open("data.zxc", "rb") as f, zxc.ZxcReader(f) as r: payload = r.read() The wrapped reader is **not** closed by :meth:`close`. """ def __init__(self, fileobj, *, checksum: bool = False, buffer_size: int = 0): super().__init__() if not hasattr(fileobj, "read"): raise TypeError("fileobj must have a .read() method") self._src = fileobj self._ds = DStream(checksum=checksum) if buffer_size <= 0: buffer_size = self._ds.in_size self._bufsize = buffer_size self._pending = b"" # decompressed bytes not yet returned self._inbuf = b"" # compressed bytes pulled from src, not yet fed self._eof_src = False # True once src.read() returned empty bytes def readable(self) -> bool: return True def readinto(self, b) -> int: if self.closed: raise ValueError("I/O operation on closed reader") n_out = len(b) if n_out == 0: return 0 while not self._pending: if self._ds.finished: return 0 if not self._inbuf and not self._eof_src: chunk = self._src.read(self._bufsize) if not chunk: self._eof_src = True else: self._inbuf = chunk produced = self._ds.decompress(self._inbuf) self._inbuf = b"" if produced: self._pending = produced break if self._eof_src: if self._ds.finished: return 0 raise OSError("zxc: input drained before footer (truncated stream)") take = min(n_out, len(self._pending)) b[:take] = self._pending[:take] self._pending = self._pending[take:] return take def close(self) -> None: if self.closed: return try: self._ds.close() finally: super().close() class ZxcWriter(_io.RawIOBase): """Compresses bytes written to it and forwards the ZXC frame to a binary file-like object. Implements the standard :class:`io.RawIOBase` interface. The frame is finalised (residual block, EOF marker, footer) when :meth:`close` is called. **You must close the writer to obtain a valid archive** — closing the wrapped file before the writer leaves a truncated frame on disk. Use the context-manager form to make this automatic:: with open("out.zxc", "wb") as f, zxc.ZxcWriter(f) as w: w.write(payload) The wrapped writer is **not** closed by :meth:`close`. """ def __init__(self, fileobj, *, level: int = LEVEL_DEFAULT, checksum: bool = False): super().__init__() if not hasattr(fileobj, "write"): raise TypeError("fileobj must have a .write() method") self._dst = fileobj self._cs = CStream(level=level, checksum=checksum) self._finalized = False def writable(self) -> bool: return True def write(self, b) -> int: if self.closed: raise ValueError("I/O operation on closed writer") if self._finalized: raise ValueError("ZxcWriter already finalised") mv = memoryview(b) if mv.nbytes == 0: return 0 chunk = self._cs.compress(bytes(mv)) if chunk: self._dst.write(chunk) return mv.nbytes def close(self) -> None: if self.closed: return try: if not self._finalized: tail = self._cs.end() if tail: self._dst.write(tail) self._finalized = True finally: self._cs.close() super().close() # Magic word identifying a ZXC file frame: little-endian 0x9CB02EF5. _ZXC_MAGIC_LE = b"\xf5\x2e\xb0\x9c" def detect_zxc(data) -> bool: """Return ``True`` if *data* starts with the ZXC file magic word. Useful for content-type sniffing in containers / object stores that need to dispatch on media type (e.g. OCI). Cheap and side-effect free; does not validate the rest of the header or the footer. """ if data is None: return False mv = memoryview(data) return len(mv) >= 4 and bytes(mv[:4]) == _ZXC_MAGIC_LEzxc-0.11.0/wrappers/python/src/zxc/__init__.pyi000066400000000000000000000063111520102567100214360ustar00rootroot00000000000000""" ZXC - High-performance lossless compression Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. SPDX-License-Identifier: BSD-3-Clause """ from typing import Protocol, Optional # ---------- constants ---------- LEVEL_FASTEST: int LEVEL_FAST: int LEVEL_DEFAULT: int LEVEL_BALANCED: int LEVEL_COMPACT: int # ---------- types ---------- class FileLike(Protocol): """File-like object with a file descriptor. Note: This excludes in-memory streams like io.BytesIO. Use real file objects or objects wrapping OS file descriptors. """ def fileno(self) -> int: ... def readable(self) -> bool: ... def writable(self) -> bool: ... # ---------- functions ---------- def compress( data: bytes, level: int = LEVEL_DEFAULT, checksum: bool = False ) -> bytes: ... def decompress( data: bytes, decompress_size: Optional[int] = None, checksum: bool = False ) -> bytes: ... def stream_compress( src: FileLike, dst: FileLike, n_threads: int = 0, level: int = LEVEL_DEFAULT, checksum: bool = False ) -> int: ... def stream_decompress( src: FileLike, dst: FileLike, n_threads: int = 0, checksum: bool = False ) -> int: ... def get_decompressed_size(data: bytes) -> int: ... def min_level() -> int: ... def max_level() -> int: ... def default_level() -> int: ... def library_version() -> str: ... # ---------- push streaming ---------- class CStream: """Push-based, single-threaded compression stream.""" def __init__(self, level: int = LEVEL_DEFAULT, checksum: bool = False, block_size: int = 0) -> None: ... def compress(self, data: bytes) -> bytes: ... def end(self) -> bytes: ... @property def in_size(self) -> int: ... @property def out_size(self) -> int: ... def close(self) -> None: ... def __enter__(self) -> "CStream": ... def __exit__(self, exc_type, exc, tb) -> bool: ... class DStream: """Push-based, single-threaded decompression stream.""" def __init__(self, checksum: bool = False) -> None: ... def decompress(self, data: bytes) -> bytes: ... @property def finished(self) -> bool: ... @property def in_size(self) -> int: ... @property def out_size(self) -> int: ... def close(self) -> None: ... def __enter__(self) -> "DStream": ... def __exit__(self, exc_type, exc, tb) -> bool: ... # ---------- io.RawIOBase adapters ---------- import io as _io from typing import IO def detect_zxc(data: bytes) -> bool: ... class ZxcReader(_io.RawIOBase): """Decompresses a ZXC frame read from a binary file-like object.""" def __init__( self, fileobj: IO[bytes], *, checksum: bool = False, buffer_size: int = 0, ) -> None: ... def readable(self) -> bool: ... def readinto(self, b: bytearray) -> int: ... def close(self) -> None: ... class ZxcWriter(_io.RawIOBase): """Compresses bytes written to it into a binary file-like object.""" def __init__( self, fileobj: IO[bytes], *, level: int = LEVEL_DEFAULT, checksum: bool = False, ) -> None: ... def writable(self) -> bool: ... def write(self, b: bytes) -> int: ... def close(self) -> None: ...zxc-0.11.0/wrappers/python/src/zxc/_zxc.c000066400000000000000000000711721520102567100202720ustar00rootroot00000000000000/* * Copyright (c) 2025-2026, Bertrand Lebonnois * All rights reserved. * * This source code is licensed under the BSD-style license found in the * LICENSE file in the root directory of this source tree. */ #define PY_SSIZE_T_CLEAN #include #include "zxc.h" #define Py_Return_Errno(err) \ do { \ PyErr_SetFromErrno(err); \ return NULL; \ } while (0) #define Py_Return_Err(err, str) \ do { \ PyErr_SetString(err, str); \ return NULL; \ } while (0) // ============================================================================= // Platform // ============================================================================= static inline int zxc_dup(int fd) { #ifdef _WIN32 return _dup(fd); #else return dup(fd); #endif } static inline FILE* zxc_fdopen(int fd, const char* mode) { #ifdef _WIN32 return _fdopen(fd, mode); #else return fdopen(fd, mode); #endif } static inline int zxc_close(int fd) { #ifdef _WIN32 return _close(fd); #else return close(fd); #endif } static int grow_output(uint8_t** buf, size_t* cap, size_t out_len, size_t want) { const size_t limit = (size_t)PY_SSIZE_T_MAX; if (out_len > limit || want > limit - out_len) return -1; const size_t needed = out_len + want; size_t new_cap = (*cap > limit / 2) ? limit : (*cap * 2); if (new_cap < needed) new_cap = needed; uint8_t* nb = (uint8_t*)realloc(*buf, new_cap); if (!nb) return -2; *buf = nb; *cap = new_cap; return 0; } // ============================================================================= // Wrapper functions // ============================================================================= static PyObject* pyzxc_compress(PyObject* self, PyObject* args, PyObject* kwargs); static PyObject* pyzxc_decompress(PyObject* self, PyObject* args, PyObject* kwargs); static PyObject* pyzxc_stream_compress(PyObject* self, PyObject* args, PyObject* kwargs); static PyObject* pyzxc_stream_decompress(PyObject* self, PyObject* args, PyObject* kwargs); static PyObject* pyzxc_get_decompressed_size(PyObject* self, PyObject* args, PyObject* kwargs); static PyObject* pyzxc_min_level(PyObject* self, PyObject* args); static PyObject* pyzxc_max_level(PyObject* self, PyObject* args); static PyObject* pyzxc_default_level(PyObject* self, PyObject* args); static PyObject* pyzxc_version_string(PyObject* self, PyObject* args); static PyObject* pyzxc_cstream_create(PyObject* self, PyObject* args, PyObject* kwargs); static PyObject* pyzxc_cstream_compress(PyObject* self, PyObject* args, PyObject* kwargs); static PyObject* pyzxc_cstream_end(PyObject* self, PyObject* args, PyObject* kwargs); static PyObject* pyzxc_cstream_in_size(PyObject* self, PyObject* args); static PyObject* pyzxc_cstream_out_size(PyObject* self, PyObject* args); static PyObject* pyzxc_cstream_free(PyObject* self, PyObject* args); static PyObject* pyzxc_dstream_create(PyObject* self, PyObject* args, PyObject* kwargs); static PyObject* pyzxc_dstream_decompress(PyObject* self, PyObject* args, PyObject* kwargs); static PyObject* pyzxc_dstream_finished(PyObject* self, PyObject* args); static PyObject* pyzxc_dstream_in_size(PyObject* self, PyObject* args); static PyObject* pyzxc_dstream_out_size(PyObject* self, PyObject* args); static PyObject* pyzxc_dstream_free(PyObject* self, PyObject* args); // ============================================================================= // Initialize python module // ============================================================================= PyDoc_STRVAR( zxc_doc, "ZXC: High-performance, lossless asymmetric compression.\n" "\n" "Functions:\n" " compress(data: bytes, level: int = LEVEL_DEFAULT, checksum: bool = False) -> bytes\n" " Compress a bytes object.\n" "\n" " decompress(data: bytes, decompress_size: int, checksum: bool = False) -> bytes\n" " Decompress a bytes object to its original size.\n" "\n" " stream_compress(src: file-like, dst: file-like, n_threads: int = 0, level: int = " "LEVEL_DEFAULT, checksum: bool = False) -> None\n" " Compress data from a readable file-like object to a writable file-like object.\n" "\n" " stream_decompress(src: file-like, dst: file-like, n_threads: int = 0, checksum: bool = " "False) -> None\n" " Decompress data from a readable file-like object to a writable file-like object.\n" "\n" "Notes:\n" " - File-like objects must support fileno(), readable(), and writable().\n" " - Stream functions release the GIL for multi-threaded compression/decompression.\n"); static PyMethodDef zxc_methods[] = { {"pyzxc_compress", (PyCFunction)pyzxc_compress, METH_VARARGS | METH_KEYWORDS, NULL}, {"pyzxc_decompress", (PyCFunction)pyzxc_decompress, METH_VARARGS | METH_KEYWORDS, NULL}, {"pyzxc_stream_compress", (PyCFunction)pyzxc_stream_compress, METH_VARARGS | METH_KEYWORDS, NULL}, {"pyzxc_stream_decompress", (PyCFunction)pyzxc_stream_decompress, METH_VARARGS | METH_KEYWORDS, NULL}, {"pyzxc_get_decompressed_size", (PyCFunction)pyzxc_get_decompressed_size, METH_VARARGS | METH_KEYWORDS, NULL}, {"pyzxc_min_level", (PyCFunction)pyzxc_min_level, METH_NOARGS, NULL}, {"pyzxc_max_level", (PyCFunction)pyzxc_max_level, METH_NOARGS, NULL}, {"pyzxc_default_level", (PyCFunction)pyzxc_default_level, METH_NOARGS, NULL}, {"pyzxc_version_string", (PyCFunction)pyzxc_version_string, METH_NOARGS, NULL}, /* Push streaming API (single-threaded, caller-driven). */ {"pyzxc_cstream_create", (PyCFunction)pyzxc_cstream_create, METH_VARARGS | METH_KEYWORDS, NULL}, {"pyzxc_cstream_compress", (PyCFunction)pyzxc_cstream_compress, METH_VARARGS | METH_KEYWORDS, NULL}, {"pyzxc_cstream_end", (PyCFunction)pyzxc_cstream_end, METH_VARARGS | METH_KEYWORDS, NULL}, {"pyzxc_cstream_in_size", (PyCFunction)pyzxc_cstream_in_size, METH_O, NULL}, {"pyzxc_cstream_out_size", (PyCFunction)pyzxc_cstream_out_size, METH_O, NULL}, {"pyzxc_cstream_free", (PyCFunction)pyzxc_cstream_free, METH_O, NULL}, {"pyzxc_dstream_create", (PyCFunction)pyzxc_dstream_create, METH_VARARGS | METH_KEYWORDS, NULL}, {"pyzxc_dstream_decompress", (PyCFunction)pyzxc_dstream_decompress, METH_VARARGS | METH_KEYWORDS, NULL}, {"pyzxc_dstream_finished", (PyCFunction)pyzxc_dstream_finished, METH_O, NULL}, {"pyzxc_dstream_in_size", (PyCFunction)pyzxc_dstream_in_size, METH_O, NULL}, {"pyzxc_dstream_out_size", (PyCFunction)pyzxc_dstream_out_size, METH_O, NULL}, {"pyzxc_dstream_free", (PyCFunction)pyzxc_dstream_free, METH_O, NULL}, {NULL, NULL, 0, NULL} // sentinel }; static struct PyModuleDef zxc_module = {PyModuleDef_HEAD_INIT, "_zxc", zxc_doc, 0, zxc_methods}; PyMODINIT_FUNC PyInit__zxc(void) { PyObject* m = PyModule_Create(&zxc_module); if (!m) return NULL; PyModule_AddIntConstant(m, "LEVEL_FASTEST", ZXC_LEVEL_FASTEST); PyModule_AddIntConstant(m, "LEVEL_FAST", ZXC_LEVEL_FAST); PyModule_AddIntConstant(m, "LEVEL_DEFAULT", ZXC_LEVEL_DEFAULT); PyModule_AddIntConstant(m, "LEVEL_BALANCED", ZXC_LEVEL_BALANCED); PyModule_AddIntConstant(m, "LEVEL_COMPACT", ZXC_LEVEL_COMPACT); PyModule_AddIntConstant(m, "LEVEL_DENSITY", ZXC_LEVEL_DENSITY); /* Error Enums */ PyModule_AddIntConstant(m, "ERROR_MEMORY", ZXC_ERROR_MEMORY); PyModule_AddIntConstant(m, "ERROR_DST_TOO_SMALL", ZXC_ERROR_DST_TOO_SMALL); PyModule_AddIntConstant(m, "ERROR_SRC_TOO_SMALL", ZXC_ERROR_SRC_TOO_SMALL); PyModule_AddIntConstant(m, "ERROR_BAD_MAGIC", ZXC_ERROR_BAD_MAGIC); PyModule_AddIntConstant(m, "ERROR_BAD_VERSION", ZXC_ERROR_BAD_VERSION); PyModule_AddIntConstant(m, "ERROR_BAD_HEADER", ZXC_ERROR_BAD_HEADER); PyModule_AddIntConstant(m, "ERROR_BAD_CHECKSUM", ZXC_ERROR_BAD_CHECKSUM); PyModule_AddIntConstant(m, "ERROR_CORRUPT_DATA", ZXC_ERROR_CORRUPT_DATA); PyModule_AddIntConstant(m, "ERROR_BAD_OFFSET", ZXC_ERROR_BAD_OFFSET); PyModule_AddIntConstant(m, "ERROR_OVERFLOW", ZXC_ERROR_OVERFLOW); PyModule_AddIntConstant(m, "ERROR_IO", ZXC_ERROR_IO); PyModule_AddIntConstant(m, "ERROR_NULL_INPUT", ZXC_ERROR_NULL_INPUT); PyModule_AddIntConstant(m, "ERROR_BAD_BLOCK_TYPE", ZXC_ERROR_BAD_BLOCK_TYPE); PyModule_AddIntConstant(m, "ERROR_BAD_BLOCK_SIZE", ZXC_ERROR_BAD_BLOCK_SIZE); return m; } // ============================================================================= // Functions definitions // ============================================================================= static PyObject* pyzxc_compress(PyObject* self, PyObject* args, PyObject* kwargs) { Py_buffer view; int level = ZXC_LEVEL_DEFAULT; int checksum = 0; static char* kwlist[] = {"data", "level", "checksum", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*|ip", kwlist, &view, &level, &checksum)) { return NULL; } if (view.itemsize != 1) { PyBuffer_Release(&view); PyErr_SetString(PyExc_TypeError, "expected a byte buffer (itemsize==1)"); return NULL; } size_t src_size = (size_t)view.len; uint64_t bound = zxc_compress_bound(src_size); PyObject* out = PyBytes_FromStringAndSize(NULL, (Py_ssize_t)bound); if (!out) { PyBuffer_Release(&view); return NULL; } char* dst = PyBytes_AsString(out); // Return a pointer to the contents int64_t nwritten; // The number of bytes written to dst zxc_compress_opts_t copts = {0}; copts.level = level; copts.checksum_enabled = checksum; Py_BEGIN_ALLOW_THREADS nwritten = zxc_compress(view.buf, // Source buffer src_size, // Source size dst, // Destination buffer bound, // Destination capacity &copts // Options ); Py_END_ALLOW_THREADS PyBuffer_Release(&view); if (nwritten < 0) { Py_DECREF(out); Py_Return_Err(PyExc_RuntimeError, zxc_error_name((int)nwritten)); } if (nwritten == 0) { Py_DECREF(out); Py_Return_Err(PyExc_ValueError, "input is too small to be compressed"); } if (_PyBytes_Resize(&out, (Py_ssize_t)nwritten) < 0) // Realloc return NULL; return out; } static PyObject* pyzxc_get_decompressed_size(PyObject* self, PyObject* args, PyObject* kwargs) { Py_buffer view; if (!PyArg_ParseTuple(args, "y*", &view)) { return NULL; } uint64_t n = zxc_get_decompressed_size(view.buf, view.len); PyBuffer_Release(&view); return Py_BuildValue("K", n); } static PyObject* pyzxc_decompress(PyObject* self, PyObject* args, PyObject* kwargs) { Py_buffer view; int checksum = 0; Py_ssize_t decompress_size; static char* kwlist[] = {"data", "decompress_size", "checksum", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*n|p", kwlist, &view, &decompress_size, &checksum)) { return NULL; } if (view.itemsize != 1) { PyBuffer_Release(&view); PyErr_SetString(PyExc_TypeError, "expected a byte buffer (itemsize==1)"); return NULL; } size_t src_size = (size_t)view.len; PyObject* out = PyBytes_FromStringAndSize(NULL, (Py_ssize_t)decompress_size); if (!out) { PyBuffer_Release(&view); return NULL; } char* dst = PyBytes_AsString(out); // Return a pointer to the contents int64_t nwritten; // The number of bytes written to dst zxc_decompress_opts_t dopts = {0}; dopts.checksum_enabled = checksum; Py_BEGIN_ALLOW_THREADS nwritten = zxc_decompress(view.buf, // Source buffer src_size, // Source size dst, // Destination buffer decompress_size, // Destination capacity &dopts // Options ); Py_END_ALLOW_THREADS PyBuffer_Release(&view); if (nwritten < 0) { Py_DECREF(out); Py_Return_Err(PyExc_RuntimeError, zxc_error_name((int)nwritten)); } return out; } static PyObject* pyzxc_stream_compress(PyObject* self, PyObject* args, PyObject* kwargs) { PyObject *src, *dst; int nthreads = 0; int level = ZXC_LEVEL_DEFAULT; int checksum = 0; int seekable = 0; static char* kwlist[] = {"src", "dst", "n_threads", "level", "checksum", "seekable", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|iipp", kwlist, &src, &dst, &nthreads, &level, &checksum, &seekable)) { return NULL; } int src_fd = PyObject_AsFileDescriptor(src); int dst_fd = PyObject_AsFileDescriptor(dst); if (src_fd == -1 || dst_fd == -1) Py_Return_Err(PyExc_RuntimeError, "couldn't get file descriptor"); int src_dup = zxc_dup(src_fd); if (src_dup == -1) { Py_Return_Errno(PyExc_OSError); } int dst_dup = zxc_dup(dst_fd); if (dst_dup == -1) { zxc_close(src_dup); Py_Return_Errno(PyExc_OSError); } FILE* fsrc = zxc_fdopen(src_dup, "rb"); if (!fsrc) { zxc_close(src_dup); zxc_close(dst_dup); Py_Return_Errno(PyExc_OSError); } FILE* fdst = zxc_fdopen(dst_dup, "wb"); if (!fdst) { fclose(fsrc); zxc_close(dst_dup); Py_Return_Errno(PyExc_OSError); } int64_t nwritten; zxc_compress_opts_t scopts = {0}; scopts.n_threads = nthreads; scopts.level = level; scopts.checksum_enabled = checksum; scopts.seekable = seekable; Py_BEGIN_ALLOW_THREADS nwritten = zxc_stream_compress(fsrc, fdst, &scopts); Py_END_ALLOW_THREADS fclose(fdst); fclose(fsrc); if (nwritten < 0) Py_Return_Err(PyExc_RuntimeError, zxc_error_name((int)nwritten)); return Py_BuildValue("L", nwritten); } static PyObject* pyzxc_stream_decompress(PyObject* self, PyObject* args, PyObject* kwargs) { PyObject *src, *dst; int nthreads = 0; int checksum = 0; static char* kwlist[] = {"src", "dst", "n_threads", "checksum", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|ip", kwlist, &src, &dst, &nthreads, &checksum)) { return NULL; } int src_fd = PyObject_AsFileDescriptor(src); int dst_fd = PyObject_AsFileDescriptor(dst); if (src_fd == -1 || dst_fd == -1) Py_Return_Err(PyExc_RuntimeError, "couldn't get file descriptor"); int src_dup = zxc_dup(src_fd); if (src_dup == -1) { Py_Return_Errno(PyExc_OSError); } int dst_dup = zxc_dup(dst_fd); if (dst_dup == -1) { zxc_close(src_dup); Py_Return_Errno(PyExc_OSError); } FILE* fsrc = zxc_fdopen(src_dup, "rb"); if (!fsrc) { zxc_close(src_dup); zxc_close(dst_dup); Py_Return_Errno(PyExc_OSError); } FILE* fdst = zxc_fdopen(dst_dup, "wb"); if (!fdst) { fclose(fsrc); zxc_close(dst_dup); Py_Return_Errno(PyExc_OSError); } int64_t nwritten; zxc_decompress_opts_t sdopts = {0}; sdopts.n_threads = nthreads; sdopts.checksum_enabled = checksum; Py_BEGIN_ALLOW_THREADS nwritten = zxc_stream_decompress(fsrc, fdst, &sdopts); Py_END_ALLOW_THREADS fclose(fdst); fclose(fsrc); if (nwritten < 0) Py_Return_Err(PyExc_RuntimeError, zxc_error_name((int)nwritten)); return Py_BuildValue("L", nwritten); } // ============================================================================= // Library Info Helpers // ============================================================================= static PyObject* pyzxc_min_level(PyObject* self, PyObject* args) { (void)self; (void)args; return PyLong_FromLong(zxc_min_level()); } static PyObject* pyzxc_max_level(PyObject* self, PyObject* args) { (void)self; (void)args; return PyLong_FromLong(zxc_max_level()); } static PyObject* pyzxc_default_level(PyObject* self, PyObject* args) { (void)self; (void)args; return PyLong_FromLong(zxc_default_level()); } static PyObject* pyzxc_version_string(PyObject* self, PyObject* args) { (void)self; (void)args; const char* const v = zxc_version_string(); return PyUnicode_FromString(v ? v : ""); } // ============================================================================= // Push Streaming API (single-threaded, caller-driven) // ============================================================================= // // Streams are exposed as PyCapsule handles. A small Python class in // __init__.py wraps these capsules into idiomatic CStream / DStream // classes; users typically never see the capsules directly. #define ZXC_CSTREAM_CAPSULE "zxc_cstream" #define ZXC_DSTREAM_CAPSULE "zxc_dstream" typedef struct { zxc_cstream* cs; } pyzxc_cstream_holder_t; typedef struct { zxc_dstream* ds; } pyzxc_dstream_holder_t; static void cstream_capsule_destructor(PyObject* capsule) { pyzxc_cstream_holder_t* h = (pyzxc_cstream_holder_t*)PyCapsule_GetPointer(capsule, ZXC_CSTREAM_CAPSULE); if (h) { if (h->cs) zxc_cstream_free(h->cs); PyMem_Free(h); } } static void dstream_capsule_destructor(PyObject* capsule) { pyzxc_dstream_holder_t* h = (pyzxc_dstream_holder_t*)PyCapsule_GetPointer(capsule, ZXC_DSTREAM_CAPSULE); if (h) { if (h->ds) zxc_dstream_free(h->ds); PyMem_Free(h); } } static zxc_cstream* cstream_from_capsule(PyObject* capsule) { pyzxc_cstream_holder_t* h = (pyzxc_cstream_holder_t*)PyCapsule_GetPointer(capsule, ZXC_CSTREAM_CAPSULE); if (!h || !h->cs) { if (!PyErr_Occurred()) PyErr_SetString(PyExc_ValueError, "cstream is closed or invalid"); return NULL; } return h->cs; } static zxc_dstream* dstream_from_capsule(PyObject* capsule) { pyzxc_dstream_holder_t* h = (pyzxc_dstream_holder_t*)PyCapsule_GetPointer(capsule, ZXC_DSTREAM_CAPSULE); if (!h || !h->ds) { if (!PyErr_Occurred()) PyErr_SetString(PyExc_ValueError, "dstream is closed or invalid"); return NULL; } return h->ds; } static PyObject* pyzxc_cstream_create(PyObject* self, PyObject* args, PyObject* kwargs) { (void)self; int level = ZXC_LEVEL_DEFAULT; int checksum = 0; Py_ssize_t block_size = 0; static char* kwlist[] = {"level", "checksum", "block_size", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|ipn", kwlist, &level, &checksum, &block_size)) { return NULL; } if (block_size < 0) { Py_Return_Err(PyExc_ValueError, "block_size must be non-negative"); } zxc_compress_opts_t copts = {0}; copts.level = level; copts.checksum_enabled = checksum; copts.block_size = (size_t)block_size; zxc_cstream* cs = zxc_cstream_create(&copts); if (!cs) Py_Return_Err(PyExc_MemoryError, "zxc_cstream_create failed"); pyzxc_cstream_holder_t* h = (pyzxc_cstream_holder_t*)PyMem_Malloc(sizeof(*h)); if (!h) { zxc_cstream_free(cs); return PyErr_NoMemory(); } h->cs = cs; PyObject* cap = PyCapsule_New(h, ZXC_CSTREAM_CAPSULE, cstream_capsule_destructor); if (!cap) { zxc_cstream_free(cs); PyMem_Free(h); return NULL; } return cap; } /* Drains the cstream by calling zxc_cstream_compress repeatedly until the * input is fully consumed and no compressed bytes remain pending. Returns * the accumulated compressed bytes. */ static PyObject* pyzxc_cstream_compress(PyObject* self, PyObject* args, PyObject* kwargs) { (void)self; PyObject* capsule; Py_buffer view; static char* kwlist[] = {"cs", "data", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "Oy*", kwlist, &capsule, &view)) { return NULL; } zxc_cstream* cs = cstream_from_capsule(capsule); if (!cs) { PyBuffer_Release(&view); return NULL; } /* Output buffer grows on demand. Start at the suggested chunk size. */ size_t out_cap = zxc_cstream_out_size(cs); if (out_cap < 4096) out_cap = 4096; size_t out_len = 0; uint8_t* out_buf = (uint8_t*)malloc(out_cap); if (!out_buf) { PyBuffer_Release(&view); return PyErr_NoMemory(); } zxc_inbuf_t in = {.src = view.buf, .size = (size_t)view.len, .pos = 0}; int err_code = 0; int oom = 0; int overflow = 0; Py_BEGIN_ALLOW_THREADS for (;;) { /* Make sure there's room to write at least one block worth. */ size_t want = zxc_cstream_out_size(cs); if (want < 4096) want = 4096; if (out_cap - out_len < want) { int rc = grow_output(&out_buf, &out_cap, out_len, want); if (rc == -1) { overflow = 1; break; } if (rc == -2) { oom = 1; break; } } zxc_outbuf_t out = {.dst = out_buf + out_len, .size = out_cap - out_len, .pos = 0}; const int64_t r = zxc_cstream_compress(cs, &out, &in); out_len += out.pos; if (r < 0) { err_code = (int)r; break; } if (r == 0 && in.pos == in.size) break; /* fully drained */ } Py_END_ALLOW_THREADS PyBuffer_Release(&view); if (oom) PyErr_NoMemory(); else if (overflow) PyErr_SetString(PyExc_OverflowError, "compressed output exceeds PY_SSIZE_T_MAX"); else if (err_code) PyErr_SetString(PyExc_RuntimeError, zxc_error_name(err_code)); if (PyErr_Occurred()) { free(out_buf); return NULL; } PyObject* result = PyBytes_FromStringAndSize((const char*)out_buf, (Py_ssize_t)out_len); free(out_buf); return result; } /* Drains the cstream's finalisation phase: residual block + EOF + footer. */ static PyObject* pyzxc_cstream_end(PyObject* self, PyObject* args, PyObject* kwargs) { (void)self; PyObject* capsule; static char* kwlist[] = {"cs", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O", kwlist, &capsule)) return NULL; zxc_cstream* cs = cstream_from_capsule(capsule); if (!cs) return NULL; size_t out_cap = zxc_cstream_out_size(cs); if (out_cap < 4096) out_cap = 4096; size_t out_len = 0; uint8_t* out_buf = (uint8_t*)malloc(out_cap); if (!out_buf) return PyErr_NoMemory(); int err_code = 0; int oom = 0; int overflow = 0; Py_BEGIN_ALLOW_THREADS for (;;) { if (out_cap - out_len < 4096) { int rc = grow_output(&out_buf, &out_cap, out_len, 4096); if (rc == -1) { overflow = 1; break; } if (rc == -2) { oom = 1; break; } } zxc_outbuf_t out = {.dst = out_buf + out_len, .size = out_cap - out_len, .pos = 0}; const int64_t r = zxc_cstream_end(cs, &out); out_len += out.pos; if (r < 0) { err_code = (int)r; break; } if (r == 0) break; } Py_END_ALLOW_THREADS if (oom) PyErr_NoMemory(); else if (overflow) PyErr_SetString(PyExc_OverflowError, "compressed output exceeds PY_SSIZE_T_MAX"); else if (err_code) PyErr_SetString(PyExc_RuntimeError, zxc_error_name(err_code)); if (PyErr_Occurred()) { free(out_buf); return NULL; } PyObject* result = PyBytes_FromStringAndSize((const char*)out_buf, (Py_ssize_t)out_len); free(out_buf); return result; } static PyObject* pyzxc_cstream_in_size(PyObject* self, PyObject* capsule) { (void)self; zxc_cstream* cs = cstream_from_capsule(capsule); if (!cs) return NULL; return PyLong_FromSize_t(zxc_cstream_in_size(cs)); } static PyObject* pyzxc_cstream_out_size(PyObject* self, PyObject* capsule) { (void)self; zxc_cstream* cs = cstream_from_capsule(capsule); if (!cs) return NULL; return PyLong_FromSize_t(zxc_cstream_out_size(cs)); } static PyObject* pyzxc_cstream_free(PyObject* self, PyObject* capsule) { (void)self; if (!PyCapsule_IsValid(capsule, ZXC_CSTREAM_CAPSULE)) Py_RETURN_NONE; pyzxc_cstream_holder_t* h = (pyzxc_cstream_holder_t*)PyCapsule_GetPointer(capsule, ZXC_CSTREAM_CAPSULE); if (h && h->cs) { zxc_cstream_free(h->cs); h->cs = NULL; } Py_RETURN_NONE; } static PyObject* pyzxc_dstream_create(PyObject* self, PyObject* args, PyObject* kwargs) { (void)self; int checksum = 0; static char* kwlist[] = {"checksum", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|p", kwlist, &checksum)) return NULL; zxc_decompress_opts_t dopts = {0}; dopts.checksum_enabled = checksum; zxc_dstream* ds = zxc_dstream_create(&dopts); if (!ds) Py_Return_Err(PyExc_MemoryError, "zxc_dstream_create failed"); pyzxc_dstream_holder_t* h = (pyzxc_dstream_holder_t*)PyMem_Malloc(sizeof(*h)); if (!h) { zxc_dstream_free(ds); return PyErr_NoMemory(); } h->ds = ds; PyObject* cap = PyCapsule_New(h, ZXC_DSTREAM_CAPSULE, dstream_capsule_destructor); if (!cap) { zxc_dstream_free(ds); PyMem_Free(h); return NULL; } return cap; } static PyObject* pyzxc_dstream_decompress(PyObject* self, PyObject* args, PyObject* kwargs) { (void)self; PyObject* capsule; Py_buffer view; static char* kwlist[] = {"ds", "data", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "Oy*", kwlist, &capsule, &view)) { return NULL; } zxc_dstream* ds = dstream_from_capsule(capsule); if (!ds) { PyBuffer_Release(&view); return NULL; } size_t out_cap = zxc_dstream_out_size(ds); if (out_cap < 4096) out_cap = 4096; size_t out_len = 0; uint8_t* out_buf = (uint8_t*)malloc(out_cap); if (!out_buf) { PyBuffer_Release(&view); return PyErr_NoMemory(); } zxc_inbuf_t in = {.src = view.buf, .size = (size_t)view.len, .pos = 0}; int err_code = 0; int oom = 0; int overflow = 0; Py_BEGIN_ALLOW_THREADS for (;;) { size_t want = zxc_dstream_out_size(ds); if (want < 4096) want = 4096; if (out_cap - out_len < want) { int rc = grow_output(&out_buf, &out_cap, out_len, want); if (rc == -1) { overflow = 1; break; } if (rc == -2) { oom = 1; break; } } zxc_outbuf_t out = {.dst = out_buf + out_len, .size = out_cap - out_len, .pos = 0}; zxc_inbuf_t empty_in = {.src = NULL, .size = 0, .pos = 0}; zxc_inbuf_t* cur_in = (in.pos < in.size) ? &in : &empty_in; const size_t before_in = cur_in->pos; const size_t before_out = out.pos; const int64_t r = zxc_dstream_decompress(ds, &out, cur_in); out_len += out.pos; if (r < 0) { err_code = (int)r; break; } /* Keep draining even after input is exhausted; stop only when no * progress was made (no input consumed AND no output produced). */ if (cur_in->pos == before_in && out.pos == before_out) break; } Py_END_ALLOW_THREADS PyBuffer_Release(&view); if (oom) PyErr_NoMemory(); else if (overflow) PyErr_SetString(PyExc_OverflowError, "decompressed output exceeds PY_SSIZE_T_MAX"); else if (err_code) PyErr_SetString(PyExc_RuntimeError, zxc_error_name(err_code)); if (PyErr_Occurred()) { free(out_buf); return NULL; } PyObject* result = PyBytes_FromStringAndSize((const char*)out_buf, (Py_ssize_t)out_len); free(out_buf); return result; } static PyObject* pyzxc_dstream_finished(PyObject* self, PyObject* capsule) { (void)self; zxc_dstream* ds = dstream_from_capsule(capsule); if (!ds) return NULL; return PyBool_FromLong(zxc_dstream_finished(ds)); } static PyObject* pyzxc_dstream_in_size(PyObject* self, PyObject* capsule) { (void)self; zxc_dstream* ds = dstream_from_capsule(capsule); if (!ds) return NULL; return PyLong_FromSize_t(zxc_dstream_in_size(ds)); } static PyObject* pyzxc_dstream_out_size(PyObject* self, PyObject* capsule) { (void)self; zxc_dstream* ds = dstream_from_capsule(capsule); if (!ds) return NULL; return PyLong_FromSize_t(zxc_dstream_out_size(ds)); } static PyObject* pyzxc_dstream_free(PyObject* self, PyObject* capsule) { (void)self; if (!PyCapsule_IsValid(capsule, ZXC_DSTREAM_CAPSULE)) Py_RETURN_NONE; pyzxc_dstream_holder_t* h = (pyzxc_dstream_holder_t*)PyCapsule_GetPointer(capsule, ZXC_DSTREAM_CAPSULE); if (h && h->ds) { zxc_dstream_free(h->ds); h->ds = NULL; } Py_RETURN_NONE; } zxc-0.11.0/wrappers/python/tests/000077500000000000000000000000001520102567100167225ustar00rootroot00000000000000zxc-0.11.0/wrappers/python/tests/test_buffer.py000066400000000000000000000030621520102567100216050ustar00rootroot00000000000000""" ZXC - High-performance lossless compression Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. SPDX-License-Identifier: BSD-3-Clause """ import pytest import zxc @pytest.mark.parametrize("data", [ None, "string", 123, 12.5, object(), ]) def test_compress_invalid_type(data): with pytest.raises(TypeError): zxc.compress(data) @pytest.mark.parametrize( "data,corrupt_func,exc", [ (b"hello world" * 10, lambda x: x[:-1] + b"\x01", RuntimeError), (b"a" * 10, lambda x: b"", RuntimeError), ], ids=["corrupted_data", "invalid_header"], ) def test_compress_corruption(data, corrupt_func, exc): compressed = zxc.compress(data, checksum=True) corrupted = corrupt_func(compressed) with pytest.raises(exc): zxc.decompress(corrupted, len(data), checksum=True) @pytest.mark.parametrize( "data", [ b"hello world" * 10, # default b"a", # single byte b"", b"a" * 10_000_000, # large data ], ids=[ "normal_data", "single_byte", "empty", "large_10mb", ], ) def test_compress_roundtrip(data): # test all compression levels (1..LEVEL_DENSITY) for level in range(zxc.LEVEL_FASTEST, zxc.LEVEL_DENSITY + 1): compressed = zxc.compress(data, level) out_size = zxc.get_decompressed_size(compressed) decompressed = zxc.decompress(compressed, out_size) assert len(data) == len(decompressed) assert data == decompressed zxc-0.11.0/wrappers/python/tests/test_io_adapters.py000066400000000000000000000106521520102567100226310ustar00rootroot00000000000000""" ZXC - High-performance lossless compression Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. SPDX-License-Identifier: BSD-3-Clause Tests for the io.RawIOBase adapters (zxc.ZxcReader / zxc.ZxcWriter) and the zxc.detect_zxc helper. """ import io import pytest import zxc def _roundtrip(data: bytes, *, level=zxc.LEVEL_DEFAULT, checksum=False) -> bytes: sink = io.BytesIO() with zxc.ZxcWriter(sink, level=level, checksum=checksum) as w: w.write(data) sink.seek(0) with zxc.ZxcReader(sink, checksum=checksum) as r: return r.read() def test_writer_reader_small_roundtrip(): data = b"Hello io.RawIOBase bridge over ZXC!" assert _roundtrip(data) == data def test_writer_reader_large_roundtrip(): data = bytes(((i * 13) % 251 for i in range(2 * 1024 * 1024))) assert _roundtrip(data) == data def test_writer_many_small_writes(): sink = io.BytesIO() want = bytearray() with zxc.ZxcWriter(sink) as w: for i in range(4096): chunk = bytes((i & 0xFF, (i >> 8) & 0xFF, (i ^ 0x5A) & 0xFF)) want.extend(chunk) w.write(chunk) sink.seek(0) with zxc.ZxcReader(sink) as r: got = r.read() assert got == bytes(want) def test_writer_reader_with_checksum(): data = bytes((i % 251 for i in range(32 * 1024))) assert _roundtrip(data, checksum=True) == data def test_writer_buffered_wrapping(): """ZxcWriter wrapped in io.BufferedWriter — typical use case.""" sink = io.BytesIO() raw = zxc.ZxcWriter(sink) bw = io.BufferedWriter(raw, buffer_size=8192) payload = bytes((i % 256 for i in range(200_000))) bw.write(payload) bw.close() # flushes BufferedWriter, then closes ZxcWriter (finalises frame) sink.seek(0) with zxc.ZxcReader(sink) as r: assert r.read() == payload def test_reader_buffered_wrapping(): """ZxcReader wrapped in io.BufferedReader for line/iteration semantics.""" sink = io.BytesIO() text = b"line one\nline two\nline three\n" with zxc.ZxcWriter(sink) as w: w.write(text) sink.seek(0) raw = zxc.ZxcReader(sink) br = io.BufferedReader(raw, buffer_size=64) assert br.readline() == b"line one\n" assert br.readline() == b"line two\n" assert br.readline() == b"line three\n" assert br.readline() == b"" br.close() def test_writer_close_idempotent(): sink = io.BytesIO() w = zxc.ZxcWriter(sink) w.write(b"abc") w.close() w.close() # must not raise sink.seek(0) with zxc.ZxcReader(sink) as r: assert r.read() == b"abc" def test_reader_close_idempotent(): sink = io.BytesIO() with zxc.ZxcWriter(sink) as w: w.write(b"xyz") sink.seek(0) r = zxc.ZxcReader(sink) r.close() r.close() # must not raise def test_reader_truncated_frame_raises(): sink = io.BytesIO() with zxc.ZxcWriter(sink) as w: w.write(b"A" * 32_768) full = sink.getvalue() truncated = full[: len(full) // 2] src = io.BytesIO(truncated) with pytest.raises(OSError, match="truncated"): with zxc.ZxcReader(src) as r: r.read() def test_writer_underlying_not_closed(): """ZxcWriter.close() must NOT close the wrapped sink.""" sink = io.BytesIO() with zxc.ZxcWriter(sink) as w: w.write(b"hello") assert not sink.closed assert sink.tell() > 0 def test_reader_underlying_not_closed(): sink = io.BytesIO() with zxc.ZxcWriter(sink) as w: w.write(b"hello") sink.seek(0) with zxc.ZxcReader(sink) as r: r.read() assert not sink.closed # --------------------------------------------------------------------------- # detect_zxc # --------------------------------------------------------------------------- def test_detect_zxc_positive_compress(): frame = zxc.compress(b"sniff me") assert zxc.detect_zxc(frame) def test_detect_zxc_positive_writer(): sink = io.BytesIO() with zxc.ZxcWriter(sink) as w: w.write(b"hi") assert zxc.detect_zxc(sink.getvalue()) @pytest.mark.parametrize( "data", [ b"", b"\xf5\x2e\xb0", # too short b"\x00\x00\x00\x00", b"not a zxc frame at all", bytes(4), ], ) def test_detect_zxc_negative(data): assert not zxc.detect_zxc(data) def test_detect_zxc_accepts_memoryview_and_bytearray(): frame = zxc.compress(b"x") assert zxc.detect_zxc(memoryview(frame)) assert zxc.detect_zxc(bytearray(frame)) zxc-0.11.0/wrappers/python/tests/test_pstream.py000066400000000000000000000050031520102567100220040ustar00rootroot00000000000000""" ZXC - High-performance lossless compression Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. SPDX-License-Identifier: BSD-3-Clause Tests for the push streaming API (zxc.CStream / zxc.DStream). """ import pytest import zxc def _roundtrip(data: bytes, *, level=zxc.LEVEL_DEFAULT, checksum=False) -> bytes: cs = zxc.CStream(level=level, checksum=checksum) chunks = [] # Feed in 17-byte slices to exercise the buffering/state machine. step = max(1, min(17, len(data))) for i in range(0, len(data), step): chunks.append(cs.compress(data[i : i + step])) chunks.append(cs.end()) cs.close() compressed = b"".join(chunks) ds = zxc.DStream(checksum=checksum) out_chunks = [] step = max(1, min(31, len(compressed))) for i in range(0, len(compressed), step): out_chunks.append(ds.decompress(compressed[i : i + step])) assert ds.finished ds.close() return b"".join(out_chunks) def test_pstream_small_roundtrip(): data = b"Hello pstream! Round-trip through the Python push API." assert _roundtrip(data) == data def test_pstream_with_checksum(): data = bytes((i % 251 for i in range(32 * 1024))) assert _roundtrip(data, checksum=True) == data def test_pstream_multi_block(): # Larger than one default block (512 KB) to force multiple blocks. data = bytes(((i * 7) % 256 for i in range(1536 * 1024))) assert _roundtrip(data) == data def test_pstream_size_hints(): cs = zxc.CStream() assert cs.in_size > 0 assert cs.out_size > 0 cs.close() ds = zxc.DStream() assert ds.in_size > 0 assert ds.out_size > 0 assert ds.finished is False ds.close() def test_pstream_context_manager(): data = b"context manager exit closes the stream" with zxc.CStream() as cs: compressed = cs.compress(data) + cs.end() with zxc.DStream() as ds: assert ds.decompress(compressed) == data assert ds.finished def test_pstream_use_after_close(): cs = zxc.CStream() cs.close() with pytest.raises(ValueError): cs.compress(b"foo") ds = zxc.DStream() ds.close() with pytest.raises(ValueError): ds.decompress(b"foo") def test_pstream_truncated_stream_not_finished(): data = b"some data " * 1000 cs = zxc.CStream() compressed = cs.compress(data) + cs.end() cs.close() # Drop the last 5 bytes (partial footer), decoder should not finalise. ds = zxc.DStream() ds.decompress(compressed[:-5]) assert not ds.finished ds.close() zxc-0.11.0/wrappers/python/tests/test_stream.py000066400000000000000000000077161520102567100216410ustar00rootroot00000000000000""" ZXC - High-performance lossless compression Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. SPDX-License-Identifier: BSD-3-Clause """ import pytest import zxc import io @pytest.mark.parametrize( "src,dst,expected_error,match", [ ("not a file", io.BytesIO(), ValueError, "src and dst must be open file-like objects"), (io.BytesIO(b"data"), "not a file", ValueError, "src and dst must be open file-like objects"), (io.BytesIO(b"data"), io.BytesIO(), ValueError, "Source file must be readable"), (io.BytesIO(b"data"), io.BytesIO(), ValueError, "Destination file must be writable"), (None, None, None, "valid_file"), ], ids=[ "src_not_file", "dst_not_file", "src_not_readable", "dst_not_writable", "valid_file", ] ) def test_stream_invalid_src_dst(tmp_path, src, dst, expected_error, match): # Helper class to mock fileno behavior class MockFile(io.BytesIO): def fileno(self): return 1 class NotReadable(MockFile): def readable(self): return False class NotWritable(MockFile): def writable(self): return False if match == "Source file must be readable": src = NotReadable(b"data") dst = MockFile() elif match == "Destination file must be writable": src = MockFile(b"data") dst = NotWritable() elif match == "valid_file": src_path = tmp_path / "src.txt" dst_path = tmp_path / "dst.zxc" src_path.write_bytes(b"hello world") src = open(src_path, "rb") dst = open(dst_path, "wb") try: if not expected_error: _ = zxc.stream_compress(src, dst) else: with pytest.raises(expected_error, match=match): zxc.stream_compress(src, dst) finally: if hasattr(src, "close"): src.close() if hasattr(dst, "close"): dst.close() @pytest.mark.parametrize( "data,corrupt_func,exc", [ (b"hello world" * 10, lambda x: x[:-1] + b"\x01", RuntimeError), (b"a" * 10, lambda x: b"", RuntimeError), ], ids=["corrupted_data", "invalid_header"], ) def test_stream_compress_corruption(tmp_path, data, corrupt_func, exc): src_file_path = tmp_path / "src.bin" compressed_file_path = tmp_path / "compressed.zxc" decompressed_file_path = tmp_path / "decompressed.bin" src_file_path.write_bytes(data) with open(src_file_path, "rb") as src, open(compressed_file_path, "wb") as dst: zxc.stream_compress(src, dst, checksum=True) compressed_bytes = compressed_file_path.read_bytes() corrupted_bytes = corrupt_func(compressed_bytes) compressed_file_path.write_bytes(corrupted_bytes) with pytest.raises(exc): with open(compressed_file_path, "rb") as src, open(decompressed_file_path, "wb") as dst: zxc.stream_decompress(src, dst, checksum=True) @pytest.mark.parametrize("data", [ b"hello world" * 10, # normal b"a", # single byte b"", # empty b"a" * 10_000_000, # large ], ids=["normal", "single_byte", "empty", "large_10mb"]) def test_stream_roundtrip(tmp_path, data): src_file_path = tmp_path / "src.txt" compressed_file_path = tmp_path / "compressed.zxc" decompressed_file_path = tmp_path / "decompressed.txt" src_file_path.write_bytes(data) for level in range(zxc.LEVEL_FASTEST, zxc.LEVEL_DENSITY + 1): with open(src_file_path, "rb") as src, \ open(compressed_file_path, "wb") as compressed: zxc.stream_compress(src, compressed, level=level) with open(compressed_file_path, "rb") as compressed, \ open(decompressed_file_path, "wb") as decompressed: zxc.stream_decompress(compressed, decompressed) decompressed_data = decompressed_file_path.read_bytes() assert decompressed_data == data assert len(decompressed_data) == len(data)zxc-0.11.0/wrappers/rust/000077500000000000000000000000001520102567100152345ustar00rootroot00000000000000zxc-0.11.0/wrappers/rust/.gitignore000066400000000000000000000004501520102567100172230ustar00rootroot00000000000000# Rust build artifacts target/ # Cargo.lock for library crates (keep it for binaries) # Since zxc-sys and zxc are libraries, we ignore Cargo.lock Cargo.lock # Debug/release build output debug/ release/ # IDE files .vscode/ .idea/ *.swp *.swo *~ # macOS .DS_Store # Backup files *.bak *.orig zxc-0.11.0/wrappers/rust/Cargo.toml000066400000000000000000000006011520102567100171610ustar00rootroot00000000000000[workspace] members = ["zxc-sys", "zxc"] resolver = "2" [workspace.package] version = "0.11.0" authors = ["Bertrand Lebonnois"] edition = "2024" rust-version = "1.85" license = "BSD-3-Clause" repository = "https://github.com/hellobertrand/zxc" keywords = [ "zxc", "compression", "lossless", "high-performance", "data-compression", ] categories = ["compression", "encoding"] zxc-0.11.0/wrappers/rust/zxc-sys/000077500000000000000000000000001520102567100166545ustar00rootroot00000000000000zxc-0.11.0/wrappers/rust/zxc-sys/Cargo.toml000066400000000000000000000010001520102567100205730ustar00rootroot00000000000000[package] name = "zxc-compress-sys" description = "Low-level FFI bindings to the ZXC compression library" version.workspace = true authors.workspace = true edition.workspace = true license.workspace = true repository.workspace = true keywords.workspace = true categories.workspace = true links = "zxc" build = "build.rs" [lib] name = "zxc_sys" [dependencies] libc = "0.2" [build-dependencies] cc = "1.2" [features] default = [] # Enable if you want to link against a system-installed ZXC library system = [] zxc-0.11.0/wrappers/rust/zxc-sys/build.rs000066400000000000000000000332761520102567100203340ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ //! Build script for zxc-sys //! //! This script compiles the ZXC C library with Function Multi-Versioning (FMV) //! to support runtime CPU feature detection and optimized code paths. //! //! On ARM64: Compiles `_default` and `_neon` variants //! On x86_64: Compiles `_default`, `_avx2`, and `_avx512` variants use std::env; use std::fs; use std::path::{Path, PathBuf}; /// Extract version constants from zxc_constants.h fn extract_version(include_dir: &Path) -> (u32, u32, u32) { let header_path = include_dir.join("zxc_constants.h"); let content = fs::read_to_string(&header_path) .expect("Failed to read zxc_constants.h"); let mut major = None; let mut minor = None; let mut patch = None; for line in content.lines() { let trimmed = line.trim(); // Parse lines like: #define ZXC_VERSION_MAJOR 0 if trimmed.starts_with("#define") { let parts: Vec<&str> = trimmed.split_whitespace().collect(); if parts.len() >= 3 { match parts[1] { "ZXC_VERSION_MAJOR" => major = parts[2].parse().ok(), "ZXC_VERSION_MINOR" => minor = parts[2].parse().ok(), "ZXC_VERSION_PATCH" => patch = parts[2].parse().ok(), _ => {} } } } } ( major.expect("ZXC_VERSION_MAJOR not found"), minor.expect("ZXC_VERSION_MINOR not found"), patch.expect("ZXC_VERSION_PATCH not found"), ) } /// Extract compression level constants from zxc_constants.h fn extract_compression_levels(include_dir: &Path) -> (i32, i32, i32, i32, i32, i32) { let header_path = include_dir.join("zxc_constants.h"); let content = fs::read_to_string(&header_path) .expect("Failed to read zxc_constants.h"); let mut fastest = None; let mut fast = None; let mut default = None; let mut balanced = None; let mut compact = None; let mut density = None; for line in content.lines() { let trimmed = line.trim(); // Parse lines like: ZXC_LEVEL_FASTEST = 1, if trimmed.starts_with("ZXC_LEVEL_") { let parts: Vec<&str> = trimmed.split('=').collect(); if parts.len() >= 2 { let name = parts[0].trim(); // Extract number, removing comma and comments let value_str = parts[1].split(&[',', '/'][..]).next().unwrap().trim(); let value: Option = value_str.parse().ok(); match name { "ZXC_LEVEL_FASTEST" => fastest = value, "ZXC_LEVEL_FAST" => fast = value, "ZXC_LEVEL_DEFAULT" => default = value, "ZXC_LEVEL_BALANCED" => balanced = value, "ZXC_LEVEL_COMPACT" => compact = value, "ZXC_LEVEL_DENSITY" => density = value, _ => {} } } } } ( fastest.expect("ZXC_LEVEL_FASTEST not found"), fast.expect("ZXC_LEVEL_FAST not found"), default.expect("ZXC_LEVEL_DEFAULT not found"), balanced.expect("ZXC_LEVEL_BALANCED not found"), compact.expect("ZXC_LEVEL_COMPACT not found"), density.expect("ZXC_LEVEL_DENSITY not found"), ) } fn main() { // Check if we should use system library instead of compiling from source if env::var("CARGO_FEATURE_SYSTEM").is_ok() { println!("cargo:rustc-link-lib=zxc"); return; } // Path to ZXC source files let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); // We use mirrored symlinks under zxc/ to preserve relative include paths // expected by the C source code (../../include/...). // During `cargo publish`, these symlinks are followed and real files are packaged. let src_lib = manifest_dir.join("zxc/src/lib"); let include_dir = manifest_dir.join("zxc/include"); // Fallback for local development if symlinks are broken or missing let (src_lib, include_dir) = if src_lib.exists() && include_dir.exists() { (src_lib, include_dir) } else { // Try direct path from workspace root let zxc_root = manifest_dir.join("../../..").canonicalize() .expect("Failed to find ZXC root directory"); (zxc_root.join("src/lib"), zxc_root.join("include")) }; // Verify paths exist assert!( src_lib.exists(), "ZXC source directory not found: {:?}", src_lib ); assert!( include_dir.exists(), "ZXC include directory not found: {:?}", include_dir ); // Extract version from header and make it available to lib.rs let (major, minor, patch) = extract_version(&include_dir); println!("cargo:rustc-env=ZXC_VERSION_MAJOR={}", major); println!("cargo:rustc-env=ZXC_VERSION_MINOR={}", minor); println!("cargo:rustc-env=ZXC_VERSION_PATCH={}", patch); // Extract compression levels from header and make them available to lib.rs let (fastest, fast, default, balanced, compact, density) = extract_compression_levels(&include_dir); println!("cargo:rustc-env=ZXC_LEVEL_FASTEST={}", fastest); println!("cargo:rustc-env=ZXC_LEVEL_FAST={}", fast); println!("cargo:rustc-env=ZXC_LEVEL_DEFAULT={}", default); println!("cargo:rustc-env=ZXC_LEVEL_BALANCED={}", balanced); println!("cargo:rustc-env=ZXC_LEVEL_COMPACT={}", compact); println!("cargo:rustc-env=ZXC_LEVEL_DENSITY={}", density); let target = env::var("TARGET").unwrap_or_default(); let is_arm64 = target.contains("aarch64") || target.contains("arm64"); let is_x86_64 = target.contains("x86_64") || target.contains("i686"); // ========================================================================= // Core library files (common to all architectures) // ========================================================================= let mut core_build = cc::Build::new(); core_build .include(&include_dir) .include(&src_lib) .include(src_lib.join("vendors")) .define("ZXC_STATIC_DEFINE", None) .file(src_lib.join("zxc_common.c")) .file(src_lib.join("zxc_driver.c")) .file(src_lib.join("zxc_dispatch.c")) .file(src_lib.join("zxc_seekable.c")) .file(src_lib.join("zxc_pstream.c")) .opt_level(3) .warnings(false) .flag_if_supported("-pthread"); // ========================================================================= // Function Multi-Versioning: Compile variants with different suffixes // ========================================================================= // --- Default variant (baseline, always compiled) --- let mut default_compress = cc::Build::new(); default_compress .include(&include_dir) .include(&src_lib) .include(src_lib.join("vendors")) .define("ZXC_STATIC_DEFINE", None) .file(src_lib.join("zxc_compress.c")) .define("ZXC_FUNCTION_SUFFIX", "_default") .opt_level(3) .warnings(false); let mut default_decompress = cc::Build::new(); default_decompress .include(&include_dir) .include(&src_lib) .include(src_lib.join("vendors")) .define("ZXC_STATIC_DEFINE", None) .file(src_lib.join("zxc_decompress.c")) .define("ZXC_FUNCTION_SUFFIX", "_default") .opt_level(3) .warnings(false); let mut default_huffman = cc::Build::new(); default_huffman .include(&include_dir) .include(&src_lib) .include(src_lib.join("vendors")) .define("ZXC_STATIC_DEFINE", None) .file(src_lib.join("zxc_huffman.c")) .define("ZXC_FUNCTION_SUFFIX", "_default") .opt_level(3) .warnings(false); // Add architecture-specific flags to core build BEFORE compiling if is_arm64 { core_build.flag_if_supported("-march=armv8-a+crc"); } else if is_x86_64 { core_build.flag_if_supported("-msse4.2"); core_build.flag_if_supported("-mpclmul"); } core_build.compile("zxc_core"); // Compile defaults default_compress.compile("zxc_compress_default"); default_decompress.compile("zxc_decompress_default"); default_huffman.compile("zxc_huffman_default"); // --- Architecture-specific variants --- if is_arm64 { // NEON variant for ARM64 let mut neon_compress = cc::Build::new(); neon_compress .include(&include_dir) .include(&src_lib) .include(src_lib.join("vendors")) .define("ZXC_STATIC_DEFINE", None) .file(src_lib.join("zxc_compress.c")) .define("ZXC_FUNCTION_SUFFIX", "_neon") .flag_if_supported("-march=armv8-a+simd") .opt_level(3) .warnings(false); let mut neon_decompress = cc::Build::new(); neon_decompress .include(&include_dir) .include(&src_lib) .include(src_lib.join("vendors")) .define("ZXC_STATIC_DEFINE", None) .file(src_lib.join("zxc_decompress.c")) .define("ZXC_FUNCTION_SUFFIX", "_neon") .flag_if_supported("-march=armv8-a+simd") .opt_level(3) .warnings(false); let mut neon_huffman = cc::Build::new(); neon_huffman .include(&include_dir) .include(&src_lib) .include(src_lib.join("vendors")) .define("ZXC_STATIC_DEFINE", None) .file(src_lib.join("zxc_huffman.c")) .define("ZXC_FUNCTION_SUFFIX", "_neon") .flag_if_supported("-march=armv8-a+simd") .opt_level(3) .warnings(false); neon_compress.compile("zxc_compress_neon"); neon_decompress.compile("zxc_decompress_neon"); neon_huffman.compile("zxc_huffman_neon"); } else if is_x86_64 { // AVX2 variant let mut avx2_compress = cc::Build::new(); avx2_compress .include(&include_dir) .include(&src_lib) .include(src_lib.join("vendors")) .define("ZXC_STATIC_DEFINE", None) .file(src_lib.join("zxc_compress.c")) .define("ZXC_FUNCTION_SUFFIX", "_avx2") .flag_if_supported("-mavx2") .flag_if_supported("-mfma") .flag_if_supported("-mbmi2") .opt_level(3) .warnings(false); let mut avx2_decompress = cc::Build::new(); avx2_decompress .include(&include_dir) .include(&src_lib) .include(src_lib.join("vendors")) .define("ZXC_STATIC_DEFINE", None) .file(src_lib.join("zxc_decompress.c")) .define("ZXC_FUNCTION_SUFFIX", "_avx2") .flag_if_supported("-mavx2") .flag_if_supported("-mfma") .flag_if_supported("-mbmi2") .opt_level(3) .warnings(false); let mut avx2_huffman = cc::Build::new(); avx2_huffman .include(&include_dir) .include(&src_lib) .include(src_lib.join("vendors")) .define("ZXC_STATIC_DEFINE", None) .file(src_lib.join("zxc_huffman.c")) .define("ZXC_FUNCTION_SUFFIX", "_avx2") .flag_if_supported("-mavx2") .flag_if_supported("-mfma") .flag_if_supported("-mbmi2") .opt_level(3) .warnings(false); avx2_compress.compile("zxc_compress_avx2"); avx2_decompress.compile("zxc_decompress_avx2"); avx2_huffman.compile("zxc_huffman_avx2"); // AVX512 variant let mut avx512_compress = cc::Build::new(); avx512_compress .include(&include_dir) .include(&src_lib) .include(src_lib.join("vendors")) .define("ZXC_STATIC_DEFINE", None) .file(src_lib.join("zxc_compress.c")) .define("ZXC_FUNCTION_SUFFIX", "_avx512") .flag_if_supported("-mavx512f") .flag_if_supported("-mavx512bw") .flag_if_supported("-mbmi2") .opt_level(3) .warnings(false); let mut avx512_decompress = cc::Build::new(); avx512_decompress .include(&include_dir) .include(&src_lib) .include(src_lib.join("vendors")) .define("ZXC_STATIC_DEFINE", None) .file(src_lib.join("zxc_decompress.c")) .define("ZXC_FUNCTION_SUFFIX", "_avx512") .flag_if_supported("-mavx512f") .flag_if_supported("-mavx512bw") .flag_if_supported("-mbmi2") .opt_level(3) .warnings(false); let mut avx512_huffman = cc::Build::new(); avx512_huffman .include(&include_dir) .include(&src_lib) .include(src_lib.join("vendors")) .define("ZXC_STATIC_DEFINE", None) .file(src_lib.join("zxc_huffman.c")) .define("ZXC_FUNCTION_SUFFIX", "_avx512") .flag_if_supported("-mavx512f") .flag_if_supported("-mavx512bw") .flag_if_supported("-mbmi2") .opt_level(3) .warnings(false); avx512_compress.compile("zxc_compress_avx512"); avx512_decompress.compile("zxc_decompress_avx512"); avx512_huffman.compile("zxc_huffman_avx512"); } // Threading support (not needed on Windows, which uses kernel32) if !target.contains("windows") { println!("cargo:rustc-link-lib=pthread"); } // Re-run build script if source files change println!("cargo:rerun-if-changed={}", src_lib.display()); println!("cargo:rerun-if-changed={}", include_dir.display()); } zxc-0.11.0/wrappers/rust/zxc-sys/src/000077500000000000000000000000001520102567100174435ustar00rootroot00000000000000zxc-0.11.0/wrappers/rust/zxc-sys/src/lib.rs000066400000000000000000001036351520102567100205670ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ //! Low-level FFI bindings to the ZXC compression library. //! //! This crate provides raw, unsafe bindings to the ZXC C library. //! For a safe, idiomatic Rust API, use the `zxc` crate instead. //! //! # Example //! //! ```rust,ignore //! use zxc_sys::*; //! //! unsafe { //! let bound = zxc_compress_bound(1024); //! // ... allocate buffer and compress //! } //! ``` #![allow(non_camel_case_types)] #![allow(non_upper_case_globals)] use std::ffi::c_int; use std::os::raw::{c_char, c_void}; // ============================================================================= // ZXC Version Constants // ============================================================================= // Version constants - automatically extracted from zxc_constants.h by build.rs const fn parse_version(s: &str) -> u32 { let bytes = s.as_bytes(); let mut result = 0u32; let mut i = 0; while i < bytes.len() { let digit = bytes[i]; if digit >= b'0' && digit <= b'9' { result = result * 10 + (digit - b'0') as u32; } i += 1; } result } pub const ZXC_VERSION_MAJOR: u32 = parse_version(env!("ZXC_VERSION_MAJOR")); pub const ZXC_VERSION_MINOR: u32 = parse_version(env!("ZXC_VERSION_MINOR")); pub const ZXC_VERSION_PATCH: u32 = parse_version(env!("ZXC_VERSION_PATCH")); // ============================================================================= // Compression Levels // ============================================================================= // Helper function to parse integer from string literal at compile time const fn parse_i32(s: &str) -> i32 { let bytes = s.as_bytes(); let mut result = 0i32; let mut i = 0; let mut sign = 1; if i < bytes.len() && bytes[i] == b'-' { sign = -1; i += 1; } while i < bytes.len() { let digit = bytes[i]; if digit >= b'0' && digit <= b'9' { result = result * 10 + (digit - b'0') as i32; } i += 1; } result * sign } // Compression level constants - automatically extracted from zxc_constants.h by build.rs /// Fastest compression, best for real-time applications pub const ZXC_LEVEL_FASTEST: i32 = parse_i32(env!("ZXC_LEVEL_FASTEST")); /// Fast compression, good for real-time applications pub const ZXC_LEVEL_FAST: i32 = parse_i32(env!("ZXC_LEVEL_FAST")); /// Recommended: ratio > LZ4, decode speed > LZ4 pub const ZXC_LEVEL_DEFAULT: i32 = parse_i32(env!("ZXC_LEVEL_DEFAULT")); /// Good ratio, good decode speed pub const ZXC_LEVEL_BALANCED: i32 = parse_i32(env!("ZXC_LEVEL_BALANCED")); /// High density. Best for storage/firmware/assets. pub const ZXC_LEVEL_COMPACT: i32 = parse_i32(env!("ZXC_LEVEL_COMPACT")); /// Maximum density: Huffman-coded literals on top of COMPACT, /// price-based optimal LZ77 parser. Slowest compression, best ratio. pub const ZXC_LEVEL_DENSITY: i32 = parse_i32(env!("ZXC_LEVEL_DENSITY")); // ============================================================================= // Error Codes // ============================================================================= /// Success (no error) pub const ZXC_OK: i32 = 0; /// Memory allocation failure pub const ZXC_ERROR_MEMORY: i32 = -1; /// Destination buffer too small pub const ZXC_ERROR_DST_TOO_SMALL: i32 = -2; /// Source buffer too small or truncated input pub const ZXC_ERROR_SRC_TOO_SMALL: i32 = -3; /// Invalid magic word in file header pub const ZXC_ERROR_BAD_MAGIC: i32 = -4; /// Unsupported file format version pub const ZXC_ERROR_BAD_VERSION: i32 = -5; /// Corrupted or invalid header (CRC mismatch) pub const ZXC_ERROR_BAD_HEADER: i32 = -6; /// Block or global checksum verification failed pub const ZXC_ERROR_BAD_CHECKSUM: i32 = -7; /// Corrupted compressed data pub const ZXC_ERROR_CORRUPT_DATA: i32 = -8; /// Invalid match offset during decompression pub const ZXC_ERROR_BAD_OFFSET: i32 = -9; /// Buffer overflow detected during processing pub const ZXC_ERROR_OVERFLOW: i32 = -10; /// Read/write/seek failure on file pub const ZXC_ERROR_IO: i32 = -11; /// Required input pointer is NULL pub const ZXC_ERROR_NULL_INPUT: i32 = -12; /// Unknown or unexpected block type pub const ZXC_ERROR_BAD_BLOCK_TYPE: i32 = -13; /// Invalid block size pub const ZXC_ERROR_BAD_BLOCK_SIZE: i32 = -14; // ============================================================================= // Options Structs (mirroring C API) // ============================================================================= /// Compression options (mirrors `zxc_compress_opts_t` from C API). #[repr(C)] #[derive(Debug, Clone)] pub struct zxc_compress_opts_t { /// Worker thread count (0 = auto-detect CPU cores). pub n_threads: c_int, /// Compression level 1-6 (0 = default). pub level: c_int, /// Block size in bytes (0 = default 512 KB). Must be power of 2, 4 KB – 2 MB. pub block_size: usize, /// 1 to enable per-block and global checksums, 0 to disable. pub checksum_enabled: c_int, /// 1 to append a seek table for random-access decompression, 0 to disable. pub seekable: c_int, /// Progress callback (NULL to disable). pub progress_cb: *const c_void, /// User context pointer passed to progress_cb. pub user_data: *mut c_void, } impl Default for zxc_compress_opts_t { fn default() -> Self { Self { n_threads: 0, level: 0, block_size: 0, checksum_enabled: 0, seekable: 0, progress_cb: std::ptr::null(), user_data: std::ptr::null_mut(), } } } /// Decompression options (mirrors `zxc_decompress_opts_t` from C API). #[repr(C)] #[derive(Debug, Clone)] pub struct zxc_decompress_opts_t { /// Worker thread count (0 = auto-detect CPU cores). pub n_threads: c_int, /// 1 to verify per-block and global checksums, 0 to skip. pub checksum_enabled: c_int, /// Progress callback (NULL to disable). pub progress_cb: *const c_void, /// User context pointer passed to progress_cb. pub user_data: *mut c_void, } impl Default for zxc_decompress_opts_t { fn default() -> Self { Self { n_threads: 0, checksum_enabled: 0, progress_cb: std::ptr::null(), user_data: std::ptr::null_mut(), } } } // ============================================================================= // Opaque Context Handles // ============================================================================= /// Opaque compression context. Use [`zxc_create_cctx`] to create. #[repr(C)] pub struct zxc_cctx { _private: [u8; 0], } /// Opaque decompression context. Use [`zxc_create_dctx`] to create. #[repr(C)] pub struct zxc_dctx { _private: [u8; 0], } // ============================================================================= // Library Info Helpers // ============================================================================= unsafe extern "C" { /// Returns the minimum supported compression level (currently 1). pub fn zxc_min_level() -> c_int; /// Returns the maximum supported compression level (currently 5). pub fn zxc_max_level() -> c_int; /// Returns the default compression level (currently 3). pub fn zxc_default_level() -> c_int; /// Returns the library version as a null-terminated string (e.g. "0.11.0"). /// /// The returned pointer is a compile-time constant and must not be freed. pub fn zxc_version_string() -> *const c_char; } // ============================================================================= // Buffer-Based API // ============================================================================= unsafe extern "C" { /// Calculates the maximum compressed size for a given input. /// /// Useful for allocating output buffers before compression. pub fn zxc_compress_bound(input_size: usize) -> u64; /// Compresses a data buffer using the ZXC algorithm. /// /// # Safety /// /// - `src` must be a valid pointer to `src_size` bytes. /// - `dst` must be a valid pointer to at least `dst_capacity` bytes. /// - The caller must ensure no data races on the buffers. /// /// # Returns /// /// Number of bytes written to `dst` (>0 on success), or a negative error code. pub fn zxc_compress( src: *const c_void, src_size: usize, dst: *mut c_void, dst_capacity: usize, opts: *const zxc_compress_opts_t, ) -> i64; /// Decompresses a ZXC compressed buffer. /// /// # Safety /// /// - `src` must be a valid pointer to `src_size` bytes of compressed data. /// - `dst` must be a valid pointer to at least `dst_capacity` bytes. /// /// # Returns /// /// Number of decompressed bytes written to `dst` (>0 on success), or a negative error code. pub fn zxc_decompress( src: *const c_void, src_size: usize, dst: *mut c_void, dst_capacity: usize, opts: *const zxc_decompress_opts_t, ) -> i64; /// Returns the decompressed size stored in a ZXC compressed buffer. /// /// Reads the file footer without performing decompression. /// /// # Returns /// /// Original uncompressed size in bytes, or 0 if invalid. pub fn zxc_get_decompressed_size(src: *const c_void, src_size: usize) -> u64; /// Returns a human-readable name for the given error code. /// /// # Arguments /// /// * `code` - An error code from zxc_error_t (or any integer) /// /// # Returns /// /// A constant string such as "ZXC_OK" or "ZXC_ERROR_MEMORY". /// Returns "ZXC_UNKNOWN_ERROR" for unrecognized codes. pub fn zxc_error_name(code: c_int) -> *const std::os::raw::c_char; } // ============================================================================= // Block API (single block, no file framing) // ============================================================================= unsafe extern "C" { /// Returns the maximum compressed size for a single block (no file framing). pub fn zxc_compress_block_bound(input_size: usize) -> u64; /// Returns the minimum `dst_capacity` required by [`zxc_decompress_block`] /// for a block of `uncompressed_size` bytes. Accounts for the wild-copy /// tail pad used by the fast decoder. pub fn zxc_decompress_block_bound(uncompressed_size: usize) -> u64; /// Compresses a single block without file framing. /// /// Output format: `block_header(8B) + payload + optional checksum(4B)`. /// /// # Safety /// - `cctx` must be a valid pointer returned by [`zxc_create_cctx`]. /// - `src`, `dst` must point to `src_size` / `dst_capacity` bytes. pub fn zxc_compress_block( cctx: *mut zxc_cctx, src: *const c_void, src_size: usize, dst: *mut c_void, dst_capacity: usize, opts: *const zxc_compress_opts_t, ) -> i64; /// Decompresses a single block produced by [`zxc_compress_block`]. /// /// `dst_capacity` should be at least /// [`zxc_decompress_block_bound(uncompressed_size)`](zxc_decompress_block_bound) /// to enable the fast path. /// /// # Safety /// - `dctx` must be a valid pointer returned by [`zxc_create_dctx`]. pub fn zxc_decompress_block( dctx: *mut zxc_dctx, src: *const c_void, src_size: usize, dst: *mut c_void, dst_capacity: usize, opts: *const zxc_decompress_opts_t, ) -> i64; /// Strict-sized variant of [`zxc_decompress_block`]: accepts /// `dst_capacity == uncompressed_size` exactly (no tail pad required). /// Slightly slower than the fast path; output is bit-identical. /// /// # Safety /// - `dctx` must be a valid pointer returned by [`zxc_create_dctx`]. pub fn zxc_decompress_block_safe( dctx: *mut zxc_dctx, src: *const c_void, src_size: usize, dst: *mut c_void, dst_capacity: usize, opts: *const zxc_decompress_opts_t, ) -> i64; /// Estimates the memory reserved by a compression context for a single /// block of `src_size` bytes via [`zxc_compress_block`]. /// /// Covers all per-chunk working buffers (chain table, literals, /// sequence/token/offset/extras buffers) plus the fixed hash tables and /// cache-line alignment padding. At `level >= 6` the value also accounts /// for the price-based optimal parser's transient DP scratch (~18 x /// `src_size` bytes), free'd at the end of each block. /// /// Returns the estimated peak memory usage in bytes, or 0 if `src_size == 0`. pub fn zxc_estimate_cctx_size(src_size: usize, level: c_int) -> u64; } // ============================================================================= // Reusable Context API (opaque, heap-allocated) // ============================================================================= unsafe extern "C" { /// Creates a heap-allocated compression context. /// /// When `opts` is non-NULL, internal buffers are pre-allocated. /// When `opts` is NULL, allocation is deferred to first use. /// Returns NULL on allocation failure. Free with [`zxc_free_cctx`]. pub fn zxc_create_cctx(opts: *const zxc_compress_opts_t) -> *mut zxc_cctx; /// Frees a compression context. Safe to call with a null pointer. /// /// # Safety /// - `cctx` must be a pointer returned by [`zxc_create_cctx`] (or null). pub fn zxc_free_cctx(cctx: *mut zxc_cctx); /// Compresses a full file-framed buffer using a reusable context. /// /// # Safety /// - `cctx` must be a valid pointer returned by [`zxc_create_cctx`]. pub fn zxc_compress_cctx( cctx: *mut zxc_cctx, src: *const c_void, src_size: usize, dst: *mut c_void, dst_capacity: usize, opts: *const zxc_compress_opts_t, ) -> i64; /// Creates a heap-allocated decompression context. /// Returns NULL on allocation failure. Free with [`zxc_free_dctx`]. pub fn zxc_create_dctx() -> *mut zxc_dctx; /// Frees a decompression context. Safe to call with a null pointer. /// /// # Safety /// - `dctx` must be a pointer returned by [`zxc_create_dctx`] (or null). pub fn zxc_free_dctx(dctx: *mut zxc_dctx); /// Decompresses a full file-framed buffer using a reusable context. /// /// # Safety /// - `dctx` must be a valid pointer returned by [`zxc_create_dctx`]. pub fn zxc_decompress_dctx( dctx: *mut zxc_dctx, src: *const c_void, src_size: usize, dst: *mut c_void, dst_capacity: usize, opts: *const zxc_decompress_opts_t, ) -> i64; } // ============================================================================= // Streaming API (FILE-based) // ============================================================================= unsafe extern "C" { /// Compresses data from an input stream to an output stream. /// /// Uses a multi-threaded pipeline architecture for high throughput /// on large files. /// /// # Safety /// /// - `f_in` must be a valid FILE* opened in "rb" mode /// - `f_out` must be a valid FILE* opened in "wb" mode /// - File handles must remain valid for the duration of the call /// /// # Arguments /// /// * `f_in` - Input file stream /// * `f_out` - Output file stream /// * `n_threads` - Number of worker threads (0 = auto-detect CPU cores) /// * `level` - Compression level (1-6) /// * `checksum_enabled` - If non-zero, enables checksum verification /// /// # Returns /// /// Total compressed bytes written, or -1 on error. pub fn zxc_stream_compress( f_in: *mut libc::FILE, f_out: *mut libc::FILE, opts: *const zxc_compress_opts_t, ) -> i64; /// Decompresses data from an input stream to an output stream. /// /// Uses the same pipeline architecture as compression for maximum throughput. /// /// # Safety /// /// - `f_in` must be a valid FILE* opened in "rb" mode /// - `f_out` must be a valid FILE* opened in "wb" mode /// /// # Arguments /// /// * `f_in` - Input file stream (compressed data) /// * `f_out` - Output file stream (decompressed data) /// * `n_threads` - Number of worker threads (0 = auto-detect) /// * `checksum_enabled` - If non-zero, verifies checksums /// /// # Returns /// /// Total decompressed bytes written, or -1 on error. pub fn zxc_stream_decompress( f_in: *mut libc::FILE, f_out: *mut libc::FILE, opts: *const zxc_decompress_opts_t, ) -> i64; /// Returns the decompressed size stored in a ZXC compressed file. /// /// Reads the file footer to extract the original size without decompressing. /// The file position is restored after reading. /// /// # Safety /// /// - `f_in` must be a valid FILE* opened in "rb" mode /// /// # Returns /// /// Original uncompressed size in bytes, or -1 on error. pub fn zxc_stream_get_decompressed_size(f_in: *mut libc::FILE) -> i64; } // ============================================================================= // Seekable API (random-access decompression) // ============================================================================= /// Opaque handle for a seekable ZXC archive. /// /// Created by [`zxc_seekable_open`] or [`zxc_seekable_open_file`]. /// Must be freed with [`zxc_seekable_free`]. #[repr(C)] pub struct zxc_seekable { _private: [u8; 0], } unsafe extern "C" { /// Opens a seekable archive from a memory buffer. /// /// The buffer must remain valid for the lifetime of the handle. /// /// # Safety /// - `src` must be a valid pointer to `src_size` bytes. /// /// Returns NULL if the buffer is not a valid seekable archive. pub fn zxc_seekable_open(src: *const c_void, src_size: usize) -> *mut zxc_seekable; /// Opens a seekable archive from a `FILE*`. The file must be seekable /// (not stdin/pipe). The current file position is saved and restored. /// The FILE* must remain open for the lifetime of the handle. /// /// # Safety /// - `f` must be a valid FILE* opened in "rb" mode. pub fn zxc_seekable_open_file(f: *mut libc::FILE) -> *mut zxc_seekable; /// Returns the total number of data blocks in the archive (excluding EOF). pub fn zxc_seekable_get_num_blocks(s: *const zxc_seekable) -> u32; /// Returns the total decompressed size of the archive in bytes. pub fn zxc_seekable_get_decompressed_size(s: *const zxc_seekable) -> u64; /// Returns the on-disk compressed size of a specific block /// (block header + payload + optional per-block checksum). /// /// Returns 0 if `block_idx` is out of range. pub fn zxc_seekable_get_block_comp_size(s: *const zxc_seekable, block_idx: u32) -> u32; /// Returns the decompressed size of a specific block. /// /// Returns 0 if `block_idx` is out of range. pub fn zxc_seekable_get_block_decomp_size(s: *const zxc_seekable, block_idx: u32) -> u32; /// Decompresses `len` bytes starting at byte `offset` in the original /// uncompressed data. Only the blocks overlapping the requested range /// are read and decompressed. /// /// # Safety /// - `s` must be a valid handle returned by [`zxc_seekable_open`]. /// - `dst` must point to at least `dst_capacity` bytes. /// /// Returns `len` on success, or a negative `zxc_error_t` on failure. pub fn zxc_seekable_decompress_range( s: *mut zxc_seekable, dst: *mut c_void, dst_capacity: usize, offset: u64, len: usize, ) -> i64; /// Multi-threaded variant of [`zxc_seekable_decompress_range`]. /// /// Each worker owns its own decompression context and reads via `pread()` /// (POSIX) or `ReadFile()` (Windows) for lock-free concurrent I/O. /// Falls back to single-threaded mode when `n_threads <= 1` or the range /// spans a single block. /// /// # Safety /// - `s` must be a valid handle returned by [`zxc_seekable_open`]. /// - `dst` must point to at least `dst_capacity` bytes. pub fn zxc_seekable_decompress_range_mt( s: *mut zxc_seekable, dst: *mut c_void, dst_capacity: usize, offset: u64, len: usize, n_threads: c_int, ) -> i64; /// Frees a seekable handle and all associated resources. /// Safe to call with a null pointer. /// /// # Safety /// - `s` must be a pointer returned by [`zxc_seekable_open`] / /// [`zxc_seekable_open_file`], or null. pub fn zxc_seekable_free(s: *mut zxc_seekable); /// Low-level: writes a seek table (block header + entries) to `dst`. /// /// # Safety /// - `dst` must point to at least `dst_capacity` bytes. /// - `comp_sizes` must point to at least `num_blocks` entries. /// /// Returns bytes written, or a negative `zxc_error_t` on failure. pub fn zxc_write_seek_table( dst: *mut u8, dst_capacity: usize, comp_sizes: *const u32, num_blocks: u32, ) -> i64; /// Returns the encoded byte size of a seek table for `num_blocks` blocks. pub fn zxc_seek_table_size(num_blocks: u32) -> usize; } // ============================================================================= // Push Streaming API (single-threaded, caller-driven) // ============================================================================= /// Input buffer descriptor for push streaming /// (mirrors `zxc_inbuf_t` from `zxc_pstream.h`). #[repr(C)] #[derive(Debug)] pub struct zxc_inbuf_t { /// Caller-owned input bytes. pub src: *const c_void, /// Total bytes available in `src`. pub size: usize, /// Bytes already consumed by the library (in/out). pub pos: usize, } /// Output buffer descriptor for push streaming /// (mirrors `zxc_outbuf_t` from `zxc_pstream.h`). #[repr(C)] #[derive(Debug)] pub struct zxc_outbuf_t { /// Caller-owned output region. pub dst: *mut c_void, /// Total capacity available at `dst`. pub size: usize, /// Bytes already produced by the library (in/out). pub pos: usize, } /// Opaque push compression stream. Use [`zxc_cstream_create`] to create. #[repr(C)] pub struct zxc_cstream { _private: [u8; 0], } /// Opaque push decompression stream. Use [`zxc_dstream_create`] to create. #[repr(C)] pub struct zxc_dstream { _private: [u8; 0], } unsafe extern "C" { /// Creates a push compression stream. Returns NULL on allocation failure. /// Free with [`zxc_cstream_free`]. pub fn zxc_cstream_create(opts: *const zxc_compress_opts_t) -> *mut zxc_cstream; /// Releases a push compression stream. Safe to call with null. /// /// # Safety /// - `cs` must be a pointer returned by [`zxc_cstream_create`] (or null). pub fn zxc_cstream_free(cs: *mut zxc_cstream); /// Pushes input bytes into the stream and drains compressed output. /// /// Returns 0 if `in` was fully consumed and no compressed bytes remain /// pending; >0 number of bytes still pending; <0 on error. /// /// # Safety /// - `cs` must be a valid stream from [`zxc_cstream_create`]. /// - `out` and `in_` must point to valid `zxc_outbuf_t` / `zxc_inbuf_t`. pub fn zxc_cstream_compress( cs: *mut zxc_cstream, out: *mut zxc_outbuf_t, in_: *mut zxc_inbuf_t, ) -> i64; /// Finalises the stream: flushes pending data, writes EOF block + footer. /// /// Returns 0 when finalisation is complete; >0 bytes still pending; <0 /// on error. /// /// # Safety /// - `cs` must be a valid stream from [`zxc_cstream_create`]. /// - `out` must point to a valid `zxc_outbuf_t`. pub fn zxc_cstream_end(cs: *mut zxc_cstream, out: *mut zxc_outbuf_t) -> i64; /// Suggested input chunk size for best compressor throughput. pub fn zxc_cstream_in_size(cs: *const zxc_cstream) -> usize; /// Suggested output chunk size for the compressor. pub fn zxc_cstream_out_size(cs: *const zxc_cstream) -> usize; /// Creates a push decompression stream. Returns NULL on allocation /// failure. Free with [`zxc_dstream_free`]. pub fn zxc_dstream_create(opts: *const zxc_decompress_opts_t) -> *mut zxc_dstream; /// Releases a push decompression stream. Safe to call with null. /// /// # Safety /// - `ds` must be a pointer returned by [`zxc_dstream_create`] (or null). pub fn zxc_dstream_free(ds: *mut zxc_dstream); /// Pushes compressed input and drains decompressed output. /// /// Returns >0 number of decompressed bytes written this call; 0 if the /// stream is complete (DONE) or no progress is possible; <0 on error. /// /// # Safety /// - `ds` must be a valid stream from [`zxc_dstream_create`]. /// - `out` and `in_` must point to valid `zxc_outbuf_t` / `zxc_inbuf_t`. pub fn zxc_dstream_decompress( ds: *mut zxc_dstream, out: *mut zxc_outbuf_t, in_: *mut zxc_inbuf_t, ) -> i64; /// Returns 1 iff the decoder reached and validated the file footer. pub fn zxc_dstream_finished(ds: *const zxc_dstream) -> c_int; /// Suggested input chunk size for the decompressor. pub fn zxc_dstream_in_size(ds: *const zxc_dstream) -> usize; /// Suggested output chunk size for the decompressor. pub fn zxc_dstream_out_size(ds: *const zxc_dstream) -> usize; } // ============================================================================= // Sans-IO API (low-level primitives for building custom drivers) // ============================================================================= /// Compression context used by the sans-IO primitives (mirrors /// `zxc_cctx_t` from `zxc_sans_io.h`). Fields are public for advanced /// integrators; most users should prefer the opaque [`zxc_cctx`] / /// [`zxc_dctx`] handles. #[repr(C)] pub struct zxc_cctx_t { /// Hash table for LZ77 match positions (epoch|pos). pub hash_table: *mut u32, /// Split tag table for fast match rejection (8-bit tags). pub hash_tags: *mut u8, /// Chain table for collision resolution. pub chain_table: *mut u16, /// Single allocation block owner. pub memory_block: *mut c_void, /// Current epoch for lazy hash table invalidation. pub epoch: u32, /// Buffer for sequence records (packed: LL|ML|Offset). pub buf_sequences: *mut u32, /// Buffer for token sequences. pub buf_tokens: *mut u8, /// Buffer for offsets. pub buf_offsets: *mut u16, /// Buffer for extra lengths (vbytes for LL/ML). pub buf_extras: *mut u8, /// Buffer for literal bytes. pub literals: *mut u8, /// Scratch buffer for literals (RLE). pub lit_buffer: *mut u8, /// Current capacity of the scratch buffer. pub lit_buffer_cap: usize, /// Padded scratch buffer for buffer-API decompression. pub work_buf: *mut u8, /// Capacity of the work buffer. pub work_buf_cap: usize, /// 1 if checksum calculation/verification is enabled. pub checksum_enabled: c_int, /// Compression level. pub compression_level: c_int, /// Effective block size in bytes. pub chunk_size: usize, /// `log2(chunk_size)` - governs epoch_mark shift. pub offset_bits: u32, /// `(1 << offset_bits) - 1`. pub offset_mask: u32, /// `1 << (32 - offset_bits)`. pub max_epoch: u32, } /// On-disk block header (8 bytes, little-endian). #[repr(C)] #[derive(Debug, Clone, Copy)] pub struct zxc_block_header_t { /// Block type (see FORMAT.md). pub block_type: u8, /// Flags (e.g., checksum presence). pub block_flags: u8, /// Reserved for future protocol extensions. pub reserved: u8, /// 1-byte header CRC. pub header_crc: u8, /// Compressed payload size (excluding this header). pub comp_size: u32, } unsafe extern "C" { /// Initializes a ZXC compression context. /// /// # Safety /// - `ctx` must point to a valid (uninitialised) `zxc_cctx_t`. /// /// Returns `ZXC_OK` on success, or a negative error code. pub fn zxc_cctx_init( ctx: *mut zxc_cctx_t, chunk_size: usize, mode: c_int, level: c_int, checksum_enabled: c_int, ) -> c_int; /// Frees internal buffers owned by a `zxc_cctx_t`. /// /// Does NOT free the context pointer itself. /// /// # Safety /// - `ctx` must be a context previously initialised with /// [`zxc_cctx_init`] (or null). pub fn zxc_cctx_free(ctx: *mut zxc_cctx_t); /// Writes the standard ZXC file header to `dst`. /// /// # Safety /// - `dst` must point to at least `dst_capacity` bytes. /// /// Returns `ZXC_FILE_HEADER_SIZE` on success, or `ZXC_ERROR_DST_TOO_SMALL`. pub fn zxc_write_file_header( dst: *mut u8, dst_capacity: usize, chunk_size: usize, has_checksum: c_int, ) -> c_int; /// Validates and parses the ZXC file header from `src`. /// /// `out_block_size` and `out_has_checksum` may be null. /// /// # Safety /// - `src` must point to at least `src_size` bytes. pub fn zxc_read_file_header( src: *const u8, src_size: usize, out_block_size: *mut usize, out_has_checksum: *mut c_int, ) -> c_int; /// Serialises a block header into `dst` (8 bytes, little-endian). /// /// # Safety /// - `dst` must point to at least `dst_capacity` bytes. /// - `bh` must be non-null. /// /// Returns `ZXC_BLOCK_HEADER_SIZE` on success, or `ZXC_ERROR_DST_TOO_SMALL`. pub fn zxc_write_block_header( dst: *mut u8, dst_capacity: usize, bh: *const zxc_block_header_t, ) -> c_int; /// Parses a block header from `src` (endianness conversion included). /// /// # Safety /// - `src` must point to at least `src_size` bytes. /// - `bh` must be a valid (possibly uninitialised) output pointer. pub fn zxc_read_block_header( src: *const u8, src_size: usize, bh: *mut zxc_block_header_t, ) -> c_int; /// Writes the 12-byte file footer (original size + optional global hash). /// /// # Safety /// - `dst` must point to at least `dst_capacity` bytes. /// /// Returns bytes written, or `ZXC_ERROR_DST_TOO_SMALL`. pub fn zxc_write_file_footer( dst: *mut u8, dst_capacity: usize, src_size: u64, global_hash: u32, checksum_enabled: c_int, ) -> c_int; } // ============================================================================= // Tests // ============================================================================= #[cfg(test)] mod tests { use super::*; #[test] fn test_compress_bound() { unsafe { let bound = zxc_compress_bound(1024); // Should return a reasonable bound (input + overhead) assert!(bound > 1024); assert!(bound < 1024 * 2); // Should not be excessively large } } #[test] fn test_roundtrip() { // Use highly repetitive data that definitely compresses well let input: Vec = (0..4096) .map(|i| ((i % 16) as u8).wrapping_add(b'A')) .collect(); unsafe { // Allocate compression buffer let bound = zxc_compress_bound(input.len()) as usize; let mut compressed = vec![0u8; bound]; // Compress let copts = zxc_compress_opts_t { level: ZXC_LEVEL_DEFAULT, checksum_enabled: 1, ..Default::default() }; let compressed_size = zxc_compress( input.as_ptr() as *const c_void, input.len(), compressed.as_mut_ptr() as *mut c_void, compressed.len(), &copts, ); assert!(compressed_size > 0, "Compression failed"); // Highly repetitive data should compress significantly assert!((compressed_size as usize) < input.len() / 2, "Data should compress well"); // Get decompressed size let decompressed_size = zxc_get_decompressed_size( compressed.as_ptr() as *const c_void, compressed_size as usize, ); assert_eq!(decompressed_size as usize, input.len()); // Decompress let mut decompressed = vec![0u8; decompressed_size as usize]; let dopts = zxc_decompress_opts_t { checksum_enabled: 1, ..Default::default() }; let result_size = zxc_decompress( compressed.as_ptr() as *const c_void, compressed_size as usize, decompressed.as_mut_ptr() as *mut c_void, decompressed.len(), &dopts, ); assert_eq!(result_size, input.len() as i64); assert_eq!(&decompressed[..], &input[..]); } } #[test] fn test_all_levels() { let input = b"Test data for all compression levels - with some repetition \ to ensure compression works: BBBBBBBBBBBBBBBBBBBBBBBBBBBBB"; for level in [ ZXC_LEVEL_FASTEST, ZXC_LEVEL_FAST, ZXC_LEVEL_DEFAULT, ZXC_LEVEL_BALANCED, ZXC_LEVEL_COMPACT, ZXC_LEVEL_DENSITY, ] { unsafe { let bound = zxc_compress_bound(input.len()) as usize; let mut compressed = vec![0u8; bound]; let copts = zxc_compress_opts_t { level, checksum_enabled: 1, ..Default::default() }; let compressed_size = zxc_compress( input.as_ptr() as *const c_void, input.len(), compressed.as_mut_ptr() as *mut c_void, compressed.len(), &copts, ); assert!( compressed_size > 0, "Compression failed at level {}", level ); } } } } zxc-0.11.0/wrappers/rust/zxc-sys/zxc/000077500000000000000000000000001520102567100174605ustar00rootroot00000000000000zxc-0.11.0/wrappers/rust/zxc-sys/zxc/include000077700000000000000000000000001520102567100234432../../../../includeustar00rootroot00000000000000zxc-0.11.0/wrappers/rust/zxc-sys/zxc/src/000077500000000000000000000000001520102567100202475ustar00rootroot00000000000000zxc-0.11.0/wrappers/rust/zxc-sys/zxc/src/lib000077700000000000000000000000001520102567100235022../../../../../src/libustar00rootroot00000000000000zxc-0.11.0/wrappers/rust/zxc/000077500000000000000000000000001520102567100160405ustar00rootroot00000000000000zxc-0.11.0/wrappers/rust/zxc/Cargo.toml000066400000000000000000000012141520102567100177660ustar00rootroot00000000000000[package] name = "zxc-compress" description = "Safe Rust bindings to the ZXC compression library" version.workspace = true authors.workspace = true edition.workspace = true license.workspace = true repository.workspace = true keywords.workspace = true categories.workspace = true readme = "README.md" [lib] name = "zxc" [dependencies] zxc_sys = { path = "../zxc-sys", version = "0.11.0", package = "zxc-compress-sys" } thiserror = "2.0" libc = "0.2" [target.'cfg(windows)'.dependencies] windows-sys = { version = "0.61.2", features = [ "Win32_Foundation", "Win32_System_Threading", ] } [dev-dependencies] rand = "0.10" [features] default = [] zxc-0.11.0/wrappers/rust/zxc/README.md000066400000000000000000000056531520102567100173300ustar00rootroot00000000000000# zxc Safe Rust bindings to the **ZXC compression library** - a fast LZ77-based compressor optimized for high decompression speed. [![Crates.io](https://img.shields.io/crates/v/zxc.svg)](https://crates.io/crates/zxc) [![Documentation](https://docs.rs/zxc/badge.svg)](https://docs.rs/zxc) [![License](https://img.shields.io/badge/license-BSD--3--Clause-blue.svg)](LICENSE) ## Quick Start ```rust use zxc::{compress, decompress, Level}; fn main() -> Result<(), zxc::Error> { let data = b"Hello, ZXC! This is some data to compress."; // Compress (no checksum for max speed) let compressed = compress(data, Level::Default, None)?; println!("Compressed {} -> {} bytes", data.len(), compressed.len()); // Decompress let decompressed = decompress(&compressed)?; assert_eq!(&decompressed[..], &data[..]); Ok(()) } ``` ## Compression Levels | Level | Speed | Ratio | Use Case | |-------|-------|-------|----------| | `Level::Fastest` | ★★★★★ | ★★☆☆☆ | Real-time, gaming | | `Level::Fast` | ★★★★☆ | ★★★☆☆ | Network, streaming | | `Level::Default` | ★★★☆☆ | ★★★★☆ | General purpose | | `Level::Balanced` | ★★☆☆☆ | ★★★★☆ | Archives | | `Level::Compact` | ★☆☆☆☆ | ★★★★★ | Storage, firmware | | `Level::Density` | ★☆☆☆☆ | ★★★★★ | Maximum density (Huffman literals + optimal parser) | ## Features - **Fast decompression**: Optimized for read-heavy workloads - **5 compression levels**: Trade off speed vs ratio - **Optional checksums**: Disabled by default for maximum performance, enable for data integrity - **File streaming**: Multi-threaded compression/decompression for large files - **Zero-allocation API**: `compress_to` and `decompress_to` for buffer reuse - **Pure Rust API**: Safe, idiomatic interface over the C library ## Advanced Usage ### Pre-allocated Buffers ```rust use zxc::{compress_to, decompress_to, compress_bound, CompressOptions, DecompressOptions}; let data = b"Hello, world!"; // Compression let mut output = vec![0u8; compress_bound(data.len())]; let size = compress_to(data, &mut output, &CompressOptions::default())?; output.truncate(size); // Decompression let mut decompressed = vec![0u8; data.len()]; decompress_to(&output, &mut decompressed, &DecompressOptions::default())?; ``` ### Disable Checksum ```rust use zxc::{compress_with_options, decompress_with_options, CompressOptions, DecompressOptions, Level}; let opts = CompressOptions::with_level(Level::Fastest).without_checksum(); let compressed = compress_with_options(data, &opts)?; let decompressed = decompress_with_options(&compressed, &DecompressOptions::skip_checksum())?; ``` ### Query Decompressed Size ```rust use zxc::decompressed_size; if let Some(size) = decompressed_size(&compressed) { let mut buffer = vec![0u8; size]; // ... } ``` ## License BSD-3-Clause - see [LICENSE](../../LICENSE) for details. zxc-0.11.0/wrappers/rust/zxc/examples/000077500000000000000000000000001520102567100176565ustar00rootroot00000000000000zxc-0.11.0/wrappers/rust/zxc/examples/file_compression.rs000066400000000000000000000050071520102567100235660ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ //! Example demonstrating file-based streaming compression and decompression. //! //! Run with: `cargo run --example file_compression` use std::fs; use std::io::Write; fn main() -> Result<(), Box> { println!("ZXC File Streaming Example\n"); // Create a test file with compressible data let test_data: Vec = (0..1024 * 1024) // 1 MB .map(|i| ((i % 256) ^ ((i / 256) % 256)) as u8) .collect(); let input_path = "/tmp/zxc_test_input.bin"; let compressed_path = "/tmp/zxc_test_compressed.zxc"; let output_path = "/tmp/zxc_test_output.bin"; // Write test data to file { let mut file = fs::File::create(input_path)?; file.write_all(&test_data)?; } // Get original file size let original_size = fs::metadata(input_path)?.len(); println!("Original file size: {} bytes", original_size); // Compress the file with multi-threading println!("\nCompressing with auto-detected threads..."); let compressed_bytes = zxc::compress_file( input_path, compressed_path, zxc::Level::Default, None, // Auto-detect CPU cores None, // Maximum performance (no checksum) )?; println!(" Compressed bytes written: {}", compressed_bytes); // Get compressed file size let compressed_size = fs::metadata(compressed_path)?.len(); let ratio = 100.0 * compressed_size as f64 / original_size as f64; println!(" Compressed file size: {} bytes ({:.1}%)", compressed_size, ratio); // Query the decompressed size from the file let reported_size = zxc::file_decompressed_size(compressed_path)?; println!("\n Reported decompressed size: {} bytes", reported_size); // Decompress the file println!("\nDecompressing..."); let decompressed_bytes = zxc::decompress_file( compressed_path, output_path, Some(4), // Use 4 threads )?; println!(" Decompressed bytes written: {}", decompressed_bytes); // Verify the result let output_data = fs::read(output_path)?; if output_data == test_data { println!("\n✓ Data integrity verified! Files match."); } else { println!("\n✗ ERROR: Data mismatch!"); } // Cleanup fs::remove_file(input_path)?; fs::remove_file(compressed_path)?; fs::remove_file(output_path)?; println!("\nDone."); Ok(()) } zxc-0.11.0/wrappers/rust/zxc/examples/simple.rs000066400000000000000000000034321520102567100215170ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ //! Simple example demonstrating ZXC compression and decompression. //! //! Run with: `cargo run --example simple` use zxc::{compress, decompress, decompressed_size, version_string, Level}; fn main() -> Result<(), zxc::Error> { println!("ZXC Rust Wrapper v{}\n", version_string()); // Sample data with some repetition (compresses well) let original = b"Hello, ZXC! This is a demonstration of the Rust wrapper. \ Let's add some repetitive content to show compression: \ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA \ BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"; println!("Original size: {} bytes", original.len()); // Test all compression levels for level in Level::all() { let compressed = compress(original, *level, None)?; let ratio = (compressed.len() as f64 / original.len() as f64) * 100.0; println!( " Level {:?}: {} bytes ({:.1}%)", level, compressed.len(), ratio ); } // Full roundtrip demonstration println!("\nRoundtrip test:"); let compressed = compress(original, Level::Default, None)?; // Query size before decompression let size = decompressed_size(&compressed).expect("valid compressed data"); println!(" Reported decompressed size: {} bytes", size); let decompressed = decompress(&compressed)?; println!(" Actual decompressed size: {} bytes", decompressed.len()); // Verify data integrity assert_eq!(&decompressed[..], &original[..]); println!(" ✓ Data integrity verified!"); Ok(()) } zxc-0.11.0/wrappers/rust/zxc/src/000077500000000000000000000000001520102567100166275ustar00rootroot00000000000000zxc-0.11.0/wrappers/rust/zxc/src/ctx.rs000066400000000000000000000143721520102567100200020ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ //! Block API: reusable single-block compression / decompression contexts. use std::ffi::c_void; use crate::error::error_from_code; use crate::{CompressOptions, DecompressOptions, Error, Result}; /// Reusable compression context for the Block API. /// /// Eliminates per-call allocation overhead when compressing many blocks. /// Internally wraps an opaque `zxc_cctx*` freed automatically on drop. /// /// # Example /// /// ```rust,ignore /// use zxc::{Cctx, CompressOptions, Level}; /// /// let mut cctx = Cctx::new(None)?; /// let opts = CompressOptions::default().with_level(Level::Default); /// let mut out = vec![0u8; zxc::compress_block_bound(block.len()) as usize]; /// let n = cctx.compress_block(block, &mut out, &opts)?; /// ``` pub struct Cctx { inner: *mut zxc_sys::zxc_cctx, } // SAFETY: the underlying handle is opaque and the library states contexts // must not be shared between threads; `Send` is safe, `Sync` is not. unsafe impl Send for Cctx {} impl Cctx { /// Creates a new compression context. /// /// When `opts` is `Some`, internal buffers are pre-allocated with those /// parameters. When `None`, allocation is deferred to first use. pub fn new(opts: Option<&CompressOptions>) -> Result { let c_opts = opts.map(|o| zxc_sys::zxc_compress_opts_t { level: o.level as i32, checksum_enabled: o.checksum as i32, seekable: o.seekable as i32, ..Default::default() }); let ptr = unsafe { zxc_sys::zxc_create_cctx( c_opts .as_ref() .map(|o| o as *const _) .unwrap_or(std::ptr::null()), ) }; if ptr.is_null() { Err(Error::Memory) } else { Ok(Self { inner: ptr }) } } /// Compresses a single block (no file framing). /// /// Output format: 8-byte block header + payload (+ optional 4-byte checksum). /// Use [`compress_block_bound`] to size `dst`. pub fn compress_block( &mut self, src: &[u8], dst: &mut [u8], opts: &CompressOptions, ) -> Result { let copts = zxc_sys::zxc_compress_opts_t { level: opts.level as i32, checksum_enabled: opts.checksum as i32, seekable: opts.seekable as i32, ..Default::default() }; let res = unsafe { zxc_sys::zxc_compress_block( self.inner, src.as_ptr() as *const c_void, src.len(), dst.as_mut_ptr() as *mut c_void, dst.len(), &copts, ) }; if res < 0 { Err(error_from_code(res)) } else { Ok(res as usize) } } } impl Drop for Cctx { fn drop(&mut self) { unsafe { zxc_sys::zxc_free_cctx(self.inner) }; } } /// Reusable decompression context for the Block API. /// /// Internally wraps an opaque `zxc_dctx*` freed automatically on drop. pub struct Dctx { inner: *mut zxc_sys::zxc_dctx, } unsafe impl Send for Dctx {} impl Dctx { /// Creates a new decompression context. pub fn new() -> Result { let ptr = unsafe { zxc_sys::zxc_create_dctx() }; if ptr.is_null() { Err(Error::Memory) } else { Ok(Self { inner: ptr }) } } /// Decompresses a single block produced by [`Cctx::compress_block`]. /// /// `dst` should be at least [`decompress_block_bound(uncompressed_size)`] /// (`decompress_block_bound`) to enable the fast path. For strictly-sized /// destinations, use [`Dctx::decompress_block_safe`]. pub fn decompress_block( &mut self, src: &[u8], dst: &mut [u8], opts: &DecompressOptions, ) -> Result { let dopts = zxc_sys::zxc_decompress_opts_t { checksum_enabled: opts.verify_checksum as i32, ..Default::default() }; let res = unsafe { zxc_sys::zxc_decompress_block( self.inner, src.as_ptr() as *const c_void, src.len(), dst.as_mut_ptr() as *mut c_void, dst.len(), &dopts, ) }; if res < 0 { Err(error_from_code(res)) } else { Ok(res as usize) } } /// Strict-sized variant of [`Dctx::decompress_block`]: accepts /// `dst.len() == uncompressed_size` exactly (no tail pad required). /// Slightly slower than the fast path; output is bit-identical. pub fn decompress_block_safe( &mut self, src: &[u8], dst: &mut [u8], opts: &DecompressOptions, ) -> Result { let dopts = zxc_sys::zxc_decompress_opts_t { checksum_enabled: opts.verify_checksum as i32, ..Default::default() }; let res = unsafe { zxc_sys::zxc_decompress_block_safe( self.inner, src.as_ptr() as *const c_void, src.len(), dst.as_mut_ptr() as *mut c_void, dst.len(), &dopts, ) }; if res < 0 { Err(error_from_code(res)) } else { Ok(res as usize) } } } impl Drop for Dctx { fn drop(&mut self) { unsafe { zxc_sys::zxc_free_dctx(self.inner) }; } } /// Returns the maximum compressed size for a single block of `input_size` /// bytes (no file framing). pub fn compress_block_bound(input_size: usize) -> u64 { unsafe { zxc_sys::zxc_compress_block_bound(input_size) } } /// Returns the minimum destination buffer size required by /// [`Dctx::decompress_block`] for a block of `uncompressed_size` bytes. /// /// Accounts for the wild-copy tail pad used by the fast decoder. For a /// strictly-sized destination, use [`Dctx::decompress_block_safe`] instead /// and size the buffer to exactly the uncompressed length. pub fn decompress_block_bound(uncompressed_size: usize) -> u64 { unsafe { zxc_sys::zxc_decompress_block_bound(uncompressed_size) } } zxc-0.11.0/wrappers/rust/zxc/src/error.rs000066400000000000000000000063301520102567100203300ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ //! Error types and code mapping shared across the crate. use zxc_sys::{ ZXC_ERROR_BAD_BLOCK_SIZE, ZXC_ERROR_BAD_BLOCK_TYPE, ZXC_ERROR_BAD_CHECKSUM, ZXC_ERROR_BAD_HEADER, ZXC_ERROR_BAD_MAGIC, ZXC_ERROR_BAD_OFFSET, ZXC_ERROR_BAD_VERSION, ZXC_ERROR_CORRUPT_DATA, ZXC_ERROR_DST_TOO_SMALL, ZXC_ERROR_IO, ZXC_ERROR_MEMORY, ZXC_ERROR_NULL_INPUT, ZXC_ERROR_OVERFLOW, ZXC_ERROR_SRC_TOO_SMALL, }; /// Errors that can occur during ZXC operations. #[derive(Debug, Clone, thiserror::Error)] pub enum Error { /// Memory allocation failure #[error("memory allocation failed")] Memory, /// Destination buffer too small #[error("destination buffer too small")] DstTooSmall, /// Source buffer too small or truncated input #[error("source buffer too small or truncated")] SrcTooSmall, /// Invalid magic word in file header #[error("invalid magic word in header")] BadMagic, /// Unsupported file format version #[error("unsupported file format version")] BadVersion, /// Corrupted or invalid header (CRC mismatch) #[error("corrupted or invalid header")] BadHeader, /// Block or global checksum verification failed #[error("checksum verification failed")] BadChecksum, /// Corrupted compressed data #[error("corrupted compressed data")] CorruptData, /// Invalid match offset during decompression #[error("invalid match offset")] BadOffset, /// Buffer overflow detected during processing #[error("buffer overflow detected")] Overflow, /// Read/write/seek failure on file #[error("I/O error")] Io, /// Required input pointer is NULL #[error("null input pointer")] NullInput, /// Unknown or unexpected block type #[error("unknown block type")] BadBlockType, /// Invalid block size #[error("invalid block size")] BadBlockSize, /// The compressed data appears to be invalid or truncated #[error("invalid compressed data")] InvalidData, /// Unknown error code from C library #[error("unknown error (code: {0})")] Unknown(i32), } /// Convert a negative error code from the C library to a Rust [`Error`]. pub(crate) fn error_from_code(code: i64) -> Error { match code as i32 { ZXC_ERROR_MEMORY => Error::Memory, ZXC_ERROR_DST_TOO_SMALL => Error::DstTooSmall, ZXC_ERROR_SRC_TOO_SMALL => Error::SrcTooSmall, ZXC_ERROR_BAD_MAGIC => Error::BadMagic, ZXC_ERROR_BAD_VERSION => Error::BadVersion, ZXC_ERROR_BAD_HEADER => Error::BadHeader, ZXC_ERROR_BAD_CHECKSUM => Error::BadChecksum, ZXC_ERROR_CORRUPT_DATA => Error::CorruptData, ZXC_ERROR_BAD_OFFSET => Error::BadOffset, ZXC_ERROR_OVERFLOW => Error::Overflow, ZXC_ERROR_IO => Error::Io, ZXC_ERROR_NULL_INPUT => Error::NullInput, ZXC_ERROR_BAD_BLOCK_TYPE => Error::BadBlockType, ZXC_ERROR_BAD_BLOCK_SIZE => Error::BadBlockSize, _ => Error::Unknown(code as i32), } } /// Result type for ZXC operations. pub type Result = std::result::Result; zxc-0.11.0/wrappers/rust/zxc/src/file.rs000066400000000000000000000437311520102567100201240ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ //! File-based multi-threaded streaming API. use std::fs::File; use std::io; use std::path::Path; #[cfg(unix)] use std::os::unix::io::AsRawFd; use crate::error::error_from_code; use crate::{Error, Level}; /// Options for streaming compression operations. #[derive(Debug, Clone)] pub struct StreamCompressOptions { /// Compression level (default: `Level::Default`) pub level: Level, /// Number of worker threads (default: `None` = auto-detect CPU cores) pub threads: Option, /// Enable checksum for data integrity (default: `true`) pub checksum: bool, /// Enable seek table for random-access decompression (default: `false`) pub seekable: bool, } impl Default for StreamCompressOptions { fn default() -> Self { Self { level: Level::Default, threads: None, checksum: true, seekable: false, } } } impl StreamCompressOptions { /// Create options with the specified compression level. pub fn with_level(level: Level) -> Self { Self { level, ..Default::default() } } /// Set the number of worker threads. pub fn threads(mut self, n: usize) -> Self { self.threads = Some(n); self } /// Disable checksum computation. pub fn without_checksum(mut self) -> Self { self.checksum = false; self } /// Enable seek table for random-access decompression. pub fn with_seekable(mut self) -> Self { self.seekable = true; self } } /// Options for streaming decompression operations. #[derive(Debug, Clone, Default)] pub struct StreamDecompressOptions { /// Number of worker threads (default: `None` = auto-detect CPU cores) pub threads: Option, /// Verify checksum during decompression (default: `true`) pub verify_checksum: bool, } impl StreamDecompressOptions { /// Set the number of worker threads. pub fn threads(mut self, n: usize) -> Self { self.threads = Some(n); self } /// Skip checksum verification. pub fn skip_checksum(mut self) -> Self { self.verify_checksum = false; self } } /// Errors specific to the streaming file API. #[derive(Debug, thiserror::Error)] pub enum StreamError { /// I/O error during file operations #[error("I/O error: {0}")] Io(#[from] io::Error), /// Error from buffer operations #[error("buffer error: {0}")] BufferError(#[from] Error), /// Streaming compression failed #[error("stream compression failed")] CompressionFailed, /// Streaming decompression failed #[error("stream decompression failed")] DecompressionFailed, /// Invalid compressed file #[error("invalid compressed file")] InvalidFile, } /// Result type for streaming operations. pub type StreamResult = std::result::Result; /// Convert a Rust File to a C FILE* for read operations. /// /// This function duplicates the file descriptor before passing it to fdopen, /// so the returned FILE* owns its own fd and must be closed with fclose(). #[cfg(unix)] unsafe fn file_to_c_file_read(file: &File) -> *mut libc::FILE { let fd = file.as_raw_fd(); // Duplicate the fd so C FILE* has its own ownership let dup_fd = unsafe { libc::dup(fd) }; if dup_fd < 0 { return std::ptr::null_mut(); } let file_ptr = unsafe { libc::fdopen(dup_fd, c"rb".as_ptr()) }; if file_ptr.is_null() { // fdopen failed, close the duplicated fd to avoid leak unsafe { libc::close(dup_fd); } } file_ptr } /// Convert a Rust File to a C FILE* for write operations. /// /// This function duplicates the file descriptor before passing it to fdopen, /// so the returned FILE* owns its own fd and must be closed with fclose(). #[cfg(unix)] unsafe fn file_to_c_file_write(file: &File) -> *mut libc::FILE { let fd = file.as_raw_fd(); // Duplicate the fd so C FILE* has its own ownership let dup_fd = unsafe { libc::dup(fd) }; if dup_fd < 0 { return std::ptr::null_mut(); } let file_ptr = unsafe { libc::fdopen(dup_fd, c"wb".as_ptr()) }; if file_ptr.is_null() { // fdopen failed, close the duplicated fd to avoid leak unsafe { libc::close(dup_fd); } } file_ptr } /// Convert a Rust File to a C FILE* for read operations (Windows). /// /// This function duplicates the file handle before passing it to the C runtime, /// so the returned FILE* owns its own handle and must be closed with fclose(). #[cfg(windows)] unsafe fn file_to_c_file_read(file: &File) -> *mut libc::FILE { use std::os::windows::io::AsRawHandle; let handle = file.as_raw_handle(); // Duplicate the handle so C FILE* has its own ownership let mut dup_handle: *mut std::ffi::c_void = std::ptr::null_mut(); let result = unsafe { windows_sys::Win32::Foundation::DuplicateHandle( windows_sys::Win32::System::Threading::GetCurrentProcess(), handle as *mut std::ffi::c_void, windows_sys::Win32::System::Threading::GetCurrentProcess(), &mut dup_handle, 0, 0, windows_sys::Win32::Foundation::DUPLICATE_SAME_ACCESS, ) }; if result == 0 { return std::ptr::null_mut(); } let fd = unsafe { libc::open_osfhandle(dup_handle as libc::intptr_t, libc::O_RDONLY) }; if fd < 0 { // open_osfhandle failed, close the duplicated handle to avoid leak unsafe { windows_sys::Win32::Foundation::CloseHandle(dup_handle); } return std::ptr::null_mut(); } let file_ptr = unsafe { libc::fdopen(fd, c"rb".as_ptr()) }; if file_ptr.is_null() { // fdopen failed, close the fd (which will close the handle) unsafe { libc::close(fd); } } file_ptr } /// Convert a Rust File to a C FILE* for write operations (Windows). /// /// This function duplicates the file handle before passing it to the C runtime, /// so the returned FILE* owns its own handle and must be closed with fclose(). #[cfg(windows)] unsafe fn file_to_c_file_write(file: &File) -> *mut libc::FILE { use std::os::windows::io::AsRawHandle; let handle = file.as_raw_handle(); // Duplicate the handle so C FILE* has its own ownership let mut dup_handle: *mut std::ffi::c_void = std::ptr::null_mut(); let result = unsafe { windows_sys::Win32::Foundation::DuplicateHandle( windows_sys::Win32::System::Threading::GetCurrentProcess(), handle as *mut std::ffi::c_void, windows_sys::Win32::System::Threading::GetCurrentProcess(), &mut dup_handle, 0, 0, windows_sys::Win32::Foundation::DUPLICATE_SAME_ACCESS, ) }; if result == 0 { return std::ptr::null_mut(); } let fd = unsafe { libc::open_osfhandle(dup_handle as libc::intptr_t, libc::O_WRONLY) }; if fd < 0 { // open_osfhandle failed, close the duplicated handle to avoid leak unsafe { windows_sys::Win32::Foundation::CloseHandle(dup_handle); } return std::ptr::null_mut(); } let file_ptr = unsafe { libc::fdopen(fd, c"wb".as_ptr()) }; if file_ptr.is_null() { // fdopen failed, close the fd (which will close the handle) unsafe { libc::close(fd); } } file_ptr } /// Compresses a file using multi-threaded streaming. /// /// This is the recommended method for compressing large files, as it: /// - Processes data in chunks without loading the entire file into memory /// - Uses multiple CPU cores for parallel compression /// - Provides better throughput for files larger than a few MB /// /// # Arguments /// /// * `input` - Path to the input file /// * `output` - Path to the output file /// * `level` - Compression level /// * `threads` - Number of threads (`None` = auto-detect CPU cores) /// * `checksum` - Optional checksum for data integrity (`None` = disabled for maximum performance) /// /// # Example /// /// ```rust,no_run /// use zxc::{compress_file, Level}; /// /// // Maximum performance (no checksum, auto threads) /// let bytes = compress_file("input.bin", "output.zxc", Level::Default, None, None)?; /// /// // With data integrity verification /// let bytes = compress_file("input.bin", "output.zxc", Level::Default, None, Some(true))?; /// /// // Custom configuration /// let bytes = compress_file("input.bin", "output.zxc", Level::Compact, Some(4), Some(true))?; /// # Ok::<(), zxc::StreamError>(()) /// ``` pub fn compress_file>( input: P, output: P, level: Level, threads: Option, checksum: Option, ) -> StreamResult { let f_in = File::open(input)?; let f_out = File::create(output)?; let n_threads = threads.unwrap_or(0) as i32; let checksum_enabled = if checksum.unwrap_or(false) { 1 } else { 0 }; unsafe { let c_in = file_to_c_file_read(&f_in); let c_out = file_to_c_file_write(&f_out); // Check for errors and cleanup on failure if c_in.is_null() { if !c_out.is_null() { libc::fclose(c_out); } return Err(StreamError::Io(io::Error::last_os_error())); } if c_out.is_null() { libc::fclose(c_in); return Err(StreamError::Io(io::Error::last_os_error())); } let result = zxc_sys::zxc_stream_compress( c_in, c_out, &zxc_sys::zxc_compress_opts_t { n_threads, level: level as i32, checksum_enabled, ..Default::default() }, ); // Always close C FILE handles (they own duplicated fds) libc::fclose(c_in); libc::fclose(c_out); if result < 0 { Err(StreamError::BufferError(error_from_code(result))) } else { Ok(result as u64) } } } /// Decompresses a file using multi-threaded streaming. /// /// # Example /// /// ```rust,no_run /// use zxc::decompress_file; /// /// // Decompress with auto-detected thread count /// let bytes = decompress_file("compressed.zxc", "output.bin", None)?; /// println!("Decompressed {} bytes", bytes); /// # Ok::<(), zxc::StreamError>(()) /// ``` pub fn decompress_file>( input: P, output: P, threads: Option, ) -> StreamResult { let f_in = File::open(input)?; let f_out = File::create(output)?; let n_threads = threads.unwrap_or(0) as i32; let checksum_enabled = 1; // Default to verify unsafe { let c_in = file_to_c_file_read(&f_in); let c_out = file_to_c_file_write(&f_out); // Check for errors and cleanup on failure if c_in.is_null() { if !c_out.is_null() { libc::fclose(c_out); } return Err(StreamError::Io(io::Error::last_os_error())); } if c_out.is_null() { libc::fclose(c_in); return Err(StreamError::Io(io::Error::last_os_error())); } let result = zxc_sys::zxc_stream_decompress( c_in, c_out, &zxc_sys::zxc_decompress_opts_t { n_threads, checksum_enabled, ..Default::default() }, ); // Always close C FILE handles (they own duplicated fds) libc::fclose(c_in); libc::fclose(c_out); if result < 0 { Err(StreamError::BufferError(error_from_code(result))) } else { Ok(result as u64) } } } /// Returns the decompressed size stored in a compressed file. /// /// This reads the file footer without performing decompression, /// useful for pre-allocating buffers or showing progress. /// /// # Example /// /// ```rust,no_run /// use zxc::file_decompressed_size; /// /// let size = file_decompressed_size("compressed.zxc")?; /// println!("Original size: {} bytes", size); /// # Ok::<(), zxc::StreamError>(()) /// ``` pub fn file_decompressed_size>(path: P) -> StreamResult { let f = File::open(path)?; unsafe { let c_file = file_to_c_file_read(&f); if c_file.is_null() { return Err(StreamError::Io(io::Error::last_os_error())); } let result = zxc_sys::zxc_stream_get_decompressed_size(c_file); if result < 0 { Err(StreamError::InvalidFile) } else { Ok(result as u64) } } } #[cfg(test)] mod tests { use crate::*; use std::fs; use std::io::Write; fn temp_path(name: &str) -> String { let dir_name = format!("zxc_test_{}", std::process::id()); // Try the system temp directory first let mut path = std::env::temp_dir(); path.push(&dir_name); // If we can't create the directory in the system temp, fall back // to a subdirectory relative to the current working directory. // This happens on some Windows CI runners where TEMP is missing or // points to a path the process cannot access. if fs::create_dir_all(&path).is_err() { path = std::env::current_dir().expect("cannot determine current directory"); path.push(&dir_name); fs::create_dir_all(&path).expect("failed to create temp directory in current dir"); } path.push(name); path.to_string_lossy().into_owned() } #[test] fn test_file_roundtrip() { let input_path = temp_path("roundtrip_input.bin"); let compressed_path = temp_path("roundtrip_compressed.zxc"); let output_path = temp_path("roundtrip_output.bin"); // Create test data let data: Vec = (0..64 * 1024) // 64 KB .map(|i| ((i % 256) ^ ((i / 256) % 256)) as u8) .collect(); // Write test file { let mut f = fs::File::create(&input_path).unwrap(); f.write_all(&data).unwrap(); } // Compress let compressed_size = compress_file(&input_path, &compressed_path, Level::Default, None, None).unwrap(); assert!(compressed_size > 0); // Decompress let decompressed_size = decompress_file(&compressed_path, &output_path, None).unwrap(); assert_eq!(decompressed_size, data.len() as u64); // Verify content let result = fs::read(&output_path).unwrap(); assert_eq!(result, data); // Cleanup let _ = fs::remove_file(&input_path); let _ = fs::remove_file(&compressed_path); let _ = fs::remove_file(&output_path); } #[test] fn test_file_decompressed_size_query() { let input_path = temp_path("size_input.bin"); let compressed_path = temp_path("size_compressed.zxc"); // Create test data let data: Vec = (0..128 * 1024) // 128 KB .map(|i| (i % 256) as u8) .collect(); // Write and compress { let mut f = fs::File::create(&input_path).unwrap(); f.write_all(&data).unwrap(); } compress_file(&input_path, &compressed_path, Level::Default, None, None).unwrap(); // Query size let reported_size = file_decompressed_size(&compressed_path).unwrap(); assert_eq!(reported_size, data.len() as u64); // Cleanup let _ = fs::remove_file(&input_path); let _ = fs::remove_file(&compressed_path); } #[test] fn test_file_all_levels() { let input_path = temp_path("levels_input.bin"); // Create test data let data: Vec = (0..32 * 1024) // 32 KB .map(|i| ((i % 256) ^ ((i / 256) % 256)) as u8) .collect(); { let mut f = fs::File::create(&input_path).unwrap(); f.write_all(&data).unwrap(); } for level in Level::all() { let compressed_path = temp_path(&format!("levels_{:?}.zxc", level)); let output_path = temp_path(&format!("levels_{:?}_out.bin", level)); // Compress with this level compress_file(&input_path, &compressed_path, *level, Some(2), None).unwrap(); // Decompress decompress_file(&compressed_path, &output_path, Some(2)).unwrap(); // Verify let result = fs::read(&output_path).unwrap(); assert_eq!(result, data, "Data mismatch at level {:?}", level); // Cleanup let _ = fs::remove_file(&compressed_path); let _ = fs::remove_file(&output_path); } let _ = fs::remove_file(&input_path); } #[test] fn test_file_multithreaded() { let input_path = temp_path("mt_input.bin"); let compressed_path = temp_path("mt_compressed.zxc"); let output_path = temp_path("mt_output.bin"); // Create larger test data (1 MB) let data: Vec = (0..1024 * 1024) .map(|i| ((i % 256) ^ ((i / 256) % 256)) as u8) .collect(); { let mut f = fs::File::create(&input_path).unwrap(); f.write_all(&data).unwrap(); } // Test with different thread counts for threads in [1, 2, 4] { // Compress compress_file( &input_path, &compressed_path, Level::Default, Some(threads), None, ) .unwrap(); // Decompress let size = decompress_file(&compressed_path, &output_path, Some(threads)).unwrap(); assert_eq!(size, data.len() as u64); // Verify let result = fs::read(&output_path).unwrap(); assert_eq!(result, data, "Mismatch with {} threads", threads); } // Cleanup let _ = fs::remove_file(&input_path); let _ = fs::remove_file(&compressed_path); let _ = fs::remove_file(&output_path); } } zxc-0.11.0/wrappers/rust/zxc/src/lib.rs000066400000000000000000000145511520102567100177510ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ //! Safe Rust bindings to the ZXC compression library. //! //! ZXC is a fast compression library optimized for high decompression speed. //! This crate provides a safe, idiomatic Rust API. //! //! # Quick Start //! //! ```rust //! use zxc::{compress, decompress, Level}; //! //! // Compress some data //! let data = b"Hello, ZXC! This is some data to compress."; //! let compressed = compress(data, Level::Default, None).expect("compression failed"); //! //! // Decompress it back //! let decompressed = decompress(&compressed).expect("decompression failed"); //! assert_eq!(&decompressed[..], &data[..]); //! ``` //! //! # Compression Levels //! //! ZXC provides 6 compression levels trading off speed vs ratio: //! //! | Level | Speed | Ratio | Use Case | //! |-------|-------|-------|----------| //! | `Fastest` | ★★★★★ | ★★☆☆☆ | Real-time, gaming | //! | `Fast` | ★★★★☆ | ★★★☆☆ | Network, streaming | //! | `Default` | ★★★☆☆ | ★★★★☆ | General purpose | //! | `Balanced` | ★★☆☆☆ | ★★★★☆ | Archives | //! | `Compact` | ★☆☆☆☆ | ★★★★★ | Storage, firmware | //! | `Density` | ★☆☆☆☆ | ★★★★★ | Maximum density (Huffman literals + optimal parser) | //! //! # Features //! //! - **Checksum verification**: Optional, disabled by default for maximum performance //! - **Zero-copy decompression bound**: Query the output size before decompressing #![warn(missing_docs)] #![warn(rust_2018_idioms)] pub use zxc_sys::{ ZXC_LEVEL_BALANCED, ZXC_LEVEL_COMPACT, ZXC_LEVEL_DEFAULT, ZXC_LEVEL_DENSITY, ZXC_LEVEL_FAST, ZXC_LEVEL_FASTEST, ZXC_VERSION_MAJOR, ZXC_VERSION_MINOR, ZXC_VERSION_PATCH, // Error codes ZXC_OK, ZXC_ERROR_MEMORY, ZXC_ERROR_DST_TOO_SMALL, ZXC_ERROR_SRC_TOO_SMALL, ZXC_ERROR_BAD_MAGIC, ZXC_ERROR_BAD_VERSION, ZXC_ERROR_BAD_HEADER, ZXC_ERROR_BAD_CHECKSUM, ZXC_ERROR_CORRUPT_DATA, ZXC_ERROR_BAD_OFFSET, ZXC_ERROR_OVERFLOW, ZXC_ERROR_IO, ZXC_ERROR_NULL_INPUT, ZXC_ERROR_BAD_BLOCK_TYPE, ZXC_ERROR_BAD_BLOCK_SIZE, }; // ============================================================================= // Compression Levels // ============================================================================= /// Compression level presets. /// /// Higher levels produce smaller output but compress more slowly. /// Decompression speed is similar across most levels; level 6 sits a notch /// below the others because Huffman-coded literals add a per-block decode /// cost relative to RAW/RLE literals. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] #[repr(i32)] pub enum Level { /// Fastest compression, best for real-time applications (level 1) Fastest = 1, /// Fast compression, good for real-time applications (level 2) Fast = 2, /// Recommended default: ratio > LZ4, decode speed > LZ4 (level 3) #[default] Default = 3, /// Good ratio, good decode speed (level 4) Balanced = 4, /// High density: storage / firmware / assets (level 5) Compact = 5, /// Maximum density: Huffman-coded literals on top of COMPACT plus a /// price-based optimal LZ77 parser. Slowest compression, best ratio /// (level 6). Density = 6, } impl Level { /// Returns all available compression levels. pub fn all() -> &'static [Level] { &[ Level::Fastest, Level::Fast, Level::Default, Level::Balanced, Level::Compact, Level::Density, ] } } impl From for i32 { fn from(level: Level) -> i32 { level as i32 } } // ============================================================================= // Compression Options // ============================================================================= /// Options for compression operations. #[derive(Debug, Clone)] pub struct CompressOptions { /// Compression level (default: `Level::Default`) pub level: Level, /// Enable checksum for data integrity (default: `true`) pub checksum: bool, /// Enable seek table for random-access decompression (default: `false`) pub seekable: bool, } impl Default for CompressOptions { fn default() -> Self { Self { level: Level::Default, checksum: true, seekable: false, } } } impl CompressOptions { /// Create options with the specified compression level. pub fn with_level(level: Level) -> Self { Self { level, ..Default::default() } } /// Disable checksum computation for faster compression. pub fn without_checksum(mut self) -> Self { self.checksum = false; self } /// Enable seek table for random-access decompression. pub fn with_seekable(mut self) -> Self { self.seekable = true; self } } // ============================================================================= // Decompression Options // ============================================================================= /// Options for decompression operations. #[derive(Debug, Clone, Default)] pub struct DecompressOptions { /// Verify checksum during decompression (default: `true`) pub verify_checksum: bool, } impl DecompressOptions { /// Create options that skip checksum verification. pub fn skip_checksum() -> Self { Self { verify_checksum: false, } } } // ============================================================================= // Submodules // ============================================================================= mod ctx; mod error; mod file; mod oneshot; mod pstream; mod stdio; pub use error::{Error, Result}; pub use oneshot::{ compress, compress_bound, compress_to, compress_with_options, decompress, decompress_to, decompress_with_options, decompressed_size, default_level, max_level, min_level, runtime_version, version, version_string, }; pub use ctx::{compress_block_bound, decompress_block_bound, Cctx, Dctx}; pub use file::{ compress_file, decompress_file, file_decompressed_size, StreamCompressOptions, StreamDecompressOptions, StreamError, StreamResult, }; pub use pstream::{CStream, CStreamProgress, DStream, DStreamProgress}; pub use stdio::{detect_zxc, Decoder, Encoder}; zxc-0.11.0/wrappers/rust/zxc/src/oneshot.rs000066400000000000000000000337621520102567100206670ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ //! One-shot compress / decompress entry points and library version helpers. use std::ffi::c_void; use zxc_sys::{ZXC_VERSION_MAJOR, ZXC_VERSION_MINOR, ZXC_VERSION_PATCH}; use crate::error::error_from_code; use crate::{CompressOptions, DecompressOptions, Error, Level, Result}; /// Returns the maximum compressed size for an input of the given size. /// /// Use this to allocate a buffer before calling [`compress_to`]. /// /// # Example /// /// ```rust /// let bound = zxc::compress_bound(1024); /// assert!(bound > 1024); // Accounts for headers and worst-case expansion /// ``` #[inline] pub fn compress_bound(input_size: usize) -> u64 { unsafe { zxc_sys::zxc_compress_bound(input_size) } } /// Compresses data with the specified level. /// /// This is a convenience function that allocates the output buffer automatically. /// For zero-allocation usage, see [`compress_to`]. /// /// # Arguments /// /// * `data` - The data to compress /// * `level` - Compression level /// * `checksum` - Optional checksum for data integrity (`None` = disabled for maximum performance) /// /// # Example /// /// ```rust /// use zxc::{compress, Level}; /// /// let data = b"Hello, world!"; /// /// // Maximum performance (no checksum) /// let compressed = compress(data, Level::Default, None)?; /// /// // With data integrity verification /// let compressed = compress(data, Level::Default, Some(true))?; /// # Ok::<(), zxc::Error>(()) /// ``` pub fn compress(data: &[u8], level: Level, checksum: Option) -> Result> { let opts = CompressOptions { level, checksum: checksum.unwrap_or(false), seekable: false, }; compress_with_options(data, &opts) } /// Compresses data with full options control. /// /// # Example /// /// ```rust /// use zxc::{compress_with_options, CompressOptions, Level}; /// /// let data = b"Hello, world!"; /// let opts = CompressOptions::with_level(Level::Compact).without_checksum(); /// let compressed = compress_with_options(data, &opts)?; /// # Ok::<(), zxc::Error>(()) /// ``` pub fn compress_with_options(data: &[u8], options: &CompressOptions) -> Result> { let bound = compress_bound(data.len()) as usize; let mut output = Vec::with_capacity(bound); let written = unsafe { impl_compress(data, output.as_mut_ptr(), output.capacity(), options)? }; unsafe { output.set_len(written); } Ok(output) } /// Helper to handle the raw compression call. /// /// # Safety /// /// `dst_ptr` must be valid for writes up to `dst_cap` bytes. #[inline(always)] unsafe fn impl_compress( data: &[u8], dst_ptr: *mut u8, dst_cap: usize, options: &CompressOptions, ) -> Result { let written = unsafe { let copts = zxc_sys::zxc_compress_opts_t { level: options.level as i32, checksum_enabled: options.checksum as i32, seekable: options.seekable as i32, ..Default::default() }; zxc_sys::zxc_compress( data.as_ptr() as *const c_void, data.len(), dst_ptr as *mut c_void, dst_cap, &copts, ) }; if written < 0 { return Err(error_from_code(written)); } if written == 0 && !data.is_empty() { return Err(Error::InvalidData); } Ok(written as usize) } /// Compresses data into a pre-allocated buffer. /// /// Returns the number of bytes written to `output`. /// /// # Errors /// /// Returns an [`Error`] if the output buffer is too small or an internal /// error occurs. /// /// # Example /// /// ```rust /// use zxc::{compress_to, compress_bound, CompressOptions}; /// /// let data = b"Hello, world!"; /// let mut output = vec![0u8; compress_bound(data.len()) as usize]; /// let size = compress_to(data, &mut output, &CompressOptions::default())?; /// output.truncate(size); /// # Ok::<(), zxc::Error>(()) /// ``` pub fn compress_to(data: &[u8], output: &mut [u8], options: &CompressOptions) -> Result { unsafe { impl_compress(data, output.as_mut_ptr(), output.len(), options) } } /// Returns the original uncompressed size from compressed data. /// /// This reads the footer without performing decompression. /// Returns `None` if the data is invalid or truncated. /// /// # Example /// /// ```rust /// use zxc::{compress, decompressed_size, Level}; /// /// let data = b"Hello, world!"; /// let compressed = compress(data, Level::Default, None)?; /// let size = decompressed_size(&compressed); /// assert_eq!(size, Some(data.len() as u64)); /// # Ok::<(), zxc::Error>(()) /// ``` pub fn decompressed_size(compressed: &[u8]) -> Option { let size = unsafe { zxc_sys::zxc_get_decompressed_size(compressed.as_ptr() as *const c_void, compressed.len()) }; if size == 0 && !compressed.is_empty() { None } else { Some(size) } } /// Decompresses ZXC-compressed data. /// /// This is a convenience function that queries the output size and allocates /// the buffer automatically. For zero-allocation usage, see [`decompress_to`]. /// /// # Example /// /// ```rust /// use zxc::{compress, decompress, Level}; /// /// let data = b"Hello, world!"; /// let compressed = compress(data, Level::Default, None)?; /// let decompressed = decompress(&compressed)?; /// assert_eq!(&decompressed[..], &data[..]); /// # Ok::<(), zxc::Error>(()) /// ``` pub fn decompress(compressed: &[u8]) -> Result> { decompress_with_options(compressed, &DecompressOptions::default()) } /// Decompresses data with full options control. pub fn decompress_with_options( compressed: &[u8], options: &DecompressOptions, ) -> Result> { let size = decompressed_size(compressed).ok_or(Error::InvalidData)? as usize; let mut output = Vec::with_capacity(size); let written = unsafe { impl_decompress(compressed, output.as_mut_ptr(), output.capacity(), options)? }; if written != size { return Err(Error::InvalidData); } unsafe { output.set_len(written); } Ok(output) } /// Helper to handle the raw decompression call. /// /// # Safety /// /// `dst_ptr` must be valid for writes up to `dst_cap` bytes. #[inline(always)] unsafe fn impl_decompress( compressed: &[u8], dst_ptr: *mut u8, dst_cap: usize, options: &DecompressOptions, ) -> Result { let written = unsafe { let dopts = zxc_sys::zxc_decompress_opts_t { checksum_enabled: if options.verify_checksum { 1 } else { 0 }, ..Default::default() }; zxc_sys::zxc_decompress( compressed.as_ptr() as *const c_void, compressed.len(), dst_ptr as *mut c_void, dst_cap, &dopts, ) }; if written < 0 { return Err(error_from_code(written)); } if written == 0 && !compressed.is_empty() { return Err(Error::InvalidData); } Ok(written as usize) } /// Decompresses data into a pre-allocated buffer. /// /// Returns the number of bytes written to `output`. /// /// # Errors /// /// Returns an error if decompression fails due to invalid data, corruption, /// or insufficient output buffer size. pub fn decompress_to( compressed: &[u8], output: &mut [u8], options: &DecompressOptions, ) -> Result { unsafe { impl_decompress(compressed, output.as_mut_ptr(), output.len(), options) } } /// Returns the library version as a tuple (major, minor, patch). pub fn version() -> (u32, u32, u32) { (ZXC_VERSION_MAJOR, ZXC_VERSION_MINOR, ZXC_VERSION_PATCH) } /// Returns the library version as a string. /// /// This value is the compile-time version baked into the `zxc-sys` crate. /// For the runtime-linked library version (useful to detect a mismatched /// shared library), call [`runtime_version`]. pub fn version_string() -> String { format!( "{}.{}.{}", ZXC_VERSION_MAJOR, ZXC_VERSION_MINOR, ZXC_VERSION_PATCH ) } /// Returns the version string reported by the linked native library /// (e.g. `"0.10.0"`). Useful for verifying that the dynamically-linked /// libzxc matches the version `zxc-sys` was built against. pub fn runtime_version() -> &'static str { unsafe { let ptr = zxc_sys::zxc_version_string(); std::ffi::CStr::from_ptr(ptr).to_str().unwrap_or("") } } /// Returns the minimum supported compression level (currently `1`). /// /// Equivalent to [`Level::Fastest`] as an integer. pub fn min_level() -> i32 { unsafe { zxc_sys::zxc_min_level() } } /// Returns the maximum supported compression level (currently `5`). /// /// Equivalent to [`Level::Compact`] as an integer. pub fn max_level() -> i32 { unsafe { zxc_sys::zxc_max_level() } } /// Returns the default compression level (currently `3`). /// /// Equivalent to [`Level::Default`] as an integer. pub fn default_level() -> i32 { unsafe { zxc_sys::zxc_default_level() } } #[cfg(test)] mod tests { use crate::*; #[test] fn test_roundtrip() { let data = b"Hello, ZXC! This is a test of the safe Rust wrapper."; let compressed = compress(data, Level::Default, None).unwrap(); let decompressed = decompress(&compressed).unwrap(); assert_eq!(&decompressed[..], &data[..]); } #[test] fn test_all_levels() { let data = b"Test data with repetition: CCCCCCCCCCCCCCCCCCCCCCCCCCCCCC"; for level in Level::all() { let compressed = compress(data, *level, None).unwrap(); let decompressed = decompress(&compressed).unwrap(); assert_eq!( &decompressed[..], &data[..], "Roundtrip failed at level {:?}", level ); } } #[test] fn test_empty() { let data: &[u8] = b""; let result = compress(data, Level::Default, None); assert!( result.is_err(), "Compressing empty data should return an error" ); } #[test] fn test_checksum_options() { let data = b"Test with and without checksum"; // With checksum let opts_with = CompressOptions::with_level(Level::Default); let compressed = compress_with_options(data, &opts_with).unwrap(); let decompressed = decompress(&compressed).unwrap(); assert_eq!(&decompressed[..], &data[..]); // Without checksum let opts_without = CompressOptions::with_level(Level::Fast).without_checksum(); let compressed = compress_with_options(data, &opts_without).unwrap(); let decompressed = decompress_with_options(&compressed, &DecompressOptions::skip_checksum()).unwrap(); assert_eq!(&decompressed[..], &data[..]); } #[test] fn test_decompressed_size() { let data = b"Hello, world! Testing decompressed_size function."; let compressed = compress(data, Level::Default, None).unwrap(); let size = decompressed_size(&compressed); assert_eq!(size, Some(data.len() as u64)); } #[test] fn test_version() { let (major, minor, patch) = version(); let expected = format!("{}.{}.{}", major, minor, patch); assert_eq!(version_string(), expected); let cargo_version = env!("CARGO_PKG_VERSION"); assert_eq!(expected, cargo_version); } #[test] fn test_invalid_data() { let garbage = b"not valid zxc data"; let result = decompress(garbage); assert!(result.is_err()); } #[test] fn test_specific_error_codes() { // Test invalid magic - size detection should fail first and return InvalidData let invalid_magic = b"INVALID_DATA_NOT_ZXC"; let result = decompress(invalid_magic); assert!(result.is_err(), "Should fail on invalid data"); // Test truncated data - should produce specific error let data = b"Hello, world! Testing error codes with enough data to compress well."; let compressed = compress(data, Level::Default, None).unwrap(); let truncated = &compressed[..10]; // Too short to be valid match decompress(truncated) { Err(Error::InvalidData) | Err(Error::SrcTooSmall) | Err(Error::BadHeader) | Err(Error::BadMagic) => { // Any of these errors are acceptable for truncated data } other => panic!( "Expected specific error for truncated data, got: {:?}", other ), } } #[test] fn test_error_messages() { // Verify error messages are descriptive let errors = vec![ (Error::Memory, "memory allocation failed"), (Error::BadChecksum, "checksum verification failed"), (Error::CorruptData, "corrupted compressed data"), (Error::DstTooSmall, "destination buffer too small"), ]; for (error, expected_msg) in errors { let msg = error.to_string(); assert!( msg.contains(expected_msg), "Error message '{}' should contain '{}'", msg, expected_msg ); } } #[test] fn test_compress_to_buffer() { let data = b"Testing compress_to with pre-allocated buffer"; let mut output = vec![0u8; compress_bound(data.len()) as usize]; let size = compress_to(data, &mut output, &CompressOptions::default()).unwrap(); output.truncate(size); let decompressed = decompress(&output).unwrap(); assert_eq!(&decompressed[..], &data[..]); } #[test] fn test_large_data() { // 1 MB of random-ish but compressible data let data: Vec = (0..1024 * 1024) .map(|i| ((i % 256) ^ ((i / 256) % 256)) as u8) .collect(); let compressed = compress(&data, Level::Default, None).unwrap(); assert!(compressed.len() < data.len()); // Should compress let decompressed = decompress(&compressed).unwrap(); assert_eq!(decompressed, data); } } zxc-0.11.0/wrappers/rust/zxc/src/pstream.rs000066400000000000000000000277161520102567100206650ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ //! Push streaming API: single-threaded, caller-driven compression / //! decompression streams. The safe Rust counterpart of `zxc_cstream` / //! `zxc_dstream`. use std::ffi::c_void; use crate::error::error_from_code; use crate::{CompressOptions, DecompressOptions, Error, Result}; /// Reports how a single [`CStream::compress`] / [`CStream::end`] call /// progressed. /// /// `pending == 0` means input was fully consumed and no output is staged /// internally; positive values indicate the caller should drain `output` /// and call again. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct CStreamProgress { /// Bytes consumed from `input` this call. pub consumed: usize, /// Bytes written into `output` this call. pub produced: usize, /// Bytes still pending in the internal staging area; drain `output` /// and call again to make progress. pub pending: u64, } /// Reports how a single [`DStream::decompress`] call progressed. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct DStreamProgress { /// Bytes consumed from `input` this call. pub consumed: usize, /// Bytes written into `output` this call. pub produced: usize, /// `true` once the decoder has reached and validated the file footer. pub finished: bool, } /// Push compression stream — the safe Rust counterpart of `zxc_cstream`. /// /// Use this when you cannot block on a `FILE*` (event loops, async runtimes, /// network protocols, callback-driven libraries). The stream is single- /// threaded; for parallel file-to-file compression, use [`crate::compress_file`]. /// /// # Example /// /// ```rust,no_run /// use zxc::{CStream, CompressOptions, Level}; /// /// let mut cs = CStream::new(Some(&CompressOptions::with_level(Level::Default)))?; /// /// let mut out = vec![0u8; cs.out_size()]; /// let mut sink: Vec = Vec::new(); /// /// for chunk in source_chunks() { /// let mut cursor = 0; /// loop { /// let p = cs.compress(&chunk[cursor..], &mut out)?; /// cursor += p.consumed; /// sink.extend_from_slice(&out[..p.produced]); /// if p.pending == 0 && cursor == chunk.len() { break; } /// } /// } /// loop { /// let p = cs.end(&mut out)?; /// sink.extend_from_slice(&out[..p.produced]); /// if p.pending == 0 { break; } /// } /// # fn source_chunks() -> Vec> { vec![] } /// # Ok::<(), zxc::Error>(()) /// ``` pub struct CStream { inner: *mut zxc_sys::zxc_cstream, } unsafe impl Send for CStream {} impl CStream { /// Creates a new push compression stream. /// /// `opts.seekable` is ignored (the push API is single-threaded and does /// not emit a seek table). Pass `None` for all defaults. pub fn new(opts: Option<&CompressOptions>) -> Result { let c_opts = opts.map(|o| zxc_sys::zxc_compress_opts_t { level: o.level as i32, checksum_enabled: o.checksum as i32, seekable: 0, ..Default::default() }); let ptr = unsafe { zxc_sys::zxc_cstream_create( c_opts .as_ref() .map(|o| o as *const _) .unwrap_or(std::ptr::null()), ) }; if ptr.is_null() { Err(Error::Memory) } else { Ok(Self { inner: ptr }) } } /// Pushes `input` into the stream and writes compressed bytes to `output`. /// /// On return, `progress.consumed` and `progress.produced` describe the /// slice ranges that were processed; `progress.pending` is the number of /// bytes still staged inside the stream (drain `output` and call again). pub fn compress(&mut self, input: &[u8], output: &mut [u8]) -> Result { let mut in_buf = zxc_sys::zxc_inbuf_t { src: input.as_ptr() as *const c_void, size: input.len(), pos: 0, }; let mut out_buf = zxc_sys::zxc_outbuf_t { dst: output.as_mut_ptr() as *mut c_void, size: output.len(), pos: 0, }; let r = unsafe { zxc_sys::zxc_cstream_compress(self.inner, &mut out_buf, &mut in_buf) }; if r < 0 { return Err(error_from_code(r)); } Ok(CStreamProgress { consumed: in_buf.pos, produced: out_buf.pos, pending: r as u64, }) } /// Finalises the stream: flushes the residual block, then writes the /// EOF block and the file footer into `output`. /// /// Call repeatedly while `pending > 0`, draining `output` between calls. pub fn end(&mut self, output: &mut [u8]) -> Result { let mut out_buf = zxc_sys::zxc_outbuf_t { dst: output.as_mut_ptr() as *mut c_void, size: output.len(), pos: 0, }; let r = unsafe { zxc_sys::zxc_cstream_end(self.inner, &mut out_buf) }; if r < 0 { return Err(error_from_code(r)); } Ok(CStreamProgress { consumed: 0, produced: out_buf.pos, pending: r as u64, }) } /// Suggested input chunk size for best throughput. pub fn in_size(&self) -> usize { unsafe { zxc_sys::zxc_cstream_in_size(self.inner) } } /// Suggested output chunk size to never trigger a partial drain. pub fn out_size(&self) -> usize { unsafe { zxc_sys::zxc_cstream_out_size(self.inner) } } } impl Drop for CStream { fn drop(&mut self) { unsafe { zxc_sys::zxc_cstream_free(self.inner) }; } } /// Push decompression stream — the safe Rust counterpart of `zxc_dstream`. /// /// # Example /// /// ```rust,no_run /// use zxc::{DStream, DecompressOptions}; /// /// let mut ds = DStream::new(None)?; /// let mut out = vec![0u8; ds.out_size()]; /// let mut sink: Vec = Vec::new(); /// /// for chunk in compressed_chunks() { /// let mut cursor = 0; /// while cursor < chunk.len() && !ds.finished() { /// let p = ds.decompress(&chunk[cursor..], &mut out)?; /// cursor += p.consumed; /// sink.extend_from_slice(&out[..p.produced]); /// if p.consumed == 0 && p.produced == 0 { break; } /// } /// } /// assert!(ds.finished()); /// # fn compressed_chunks() -> Vec> { vec![] } /// # Ok::<(), zxc::Error>(()) /// ``` pub struct DStream { inner: *mut zxc_sys::zxc_dstream, } unsafe impl Send for DStream {} impl DStream { /// Creates a new push decompression stream. pub fn new(opts: Option<&DecompressOptions>) -> Result { let c_opts = opts.map(|o| zxc_sys::zxc_decompress_opts_t { checksum_enabled: o.verify_checksum as i32, ..Default::default() }); let ptr = unsafe { zxc_sys::zxc_dstream_create( c_opts .as_ref() .map(|o| o as *const _) .unwrap_or(std::ptr::null()), ) }; if ptr.is_null() { Err(Error::Memory) } else { Ok(Self { inner: ptr }) } } /// Pushes compressed bytes from `input` and writes decompressed bytes /// to `output`. The state machine consumes file header, block headers, /// payloads, and footer transparently. pub fn decompress(&mut self, input: &[u8], output: &mut [u8]) -> Result { let mut in_buf = zxc_sys::zxc_inbuf_t { src: input.as_ptr() as *const c_void, size: input.len(), pos: 0, }; let mut out_buf = zxc_sys::zxc_outbuf_t { dst: output.as_mut_ptr() as *mut c_void, size: output.len(), pos: 0, }; let r = unsafe { zxc_sys::zxc_dstream_decompress(self.inner, &mut out_buf, &mut in_buf) }; if r < 0 { return Err(error_from_code(r)); } Ok(DStreamProgress { consumed: in_buf.pos, produced: out_buf.pos, finished: unsafe { zxc_sys::zxc_dstream_finished(self.inner) } != 0, }) } /// Returns `true` iff the decoder reached and validated the file footer. /// Useful to detect truncated streams after the input source is drained. pub fn finished(&self) -> bool { unsafe { zxc_sys::zxc_dstream_finished(self.inner) != 0 } } /// Suggested input chunk size for the decompressor. pub fn in_size(&self) -> usize { unsafe { zxc_sys::zxc_dstream_in_size(self.inner) } } /// Suggested output chunk size for the decompressor. pub fn out_size(&self) -> usize { unsafe { zxc_sys::zxc_dstream_out_size(self.inner) } } } impl Drop for DStream { fn drop(&mut self) { unsafe { zxc_sys::zxc_dstream_free(self.inner) }; } } #[cfg(test)] mod tests { use crate::*; fn pstream_roundtrip( data: &[u8], copts: Option<&CompressOptions>, dopts: Option<&DecompressOptions>, ) -> Vec { let mut cs = CStream::new(copts).expect("cstream alloc"); let mut compressed: Vec = Vec::new(); let mut out = vec![0u8; cs.out_size().max(64)]; let mut cursor = 0; while cursor < data.len() { loop { let p = cs.compress(&data[cursor..], &mut out).unwrap(); cursor += p.consumed; compressed.extend_from_slice(&out[..p.produced]); if p.pending == 0 && (p.consumed > 0 || cursor == data.len()) { break; } } } loop { let p = cs.end(&mut out).unwrap(); compressed.extend_from_slice(&out[..p.produced]); if p.pending == 0 { break; } } let mut ds = DStream::new(dopts).expect("dstream alloc"); let mut decompressed: Vec = Vec::new(); let mut dout = vec![0u8; 64 * 1024]; let mut cursor = 0; while cursor < compressed.len() && !ds.finished() { let p = ds.decompress(&compressed[cursor..], &mut dout).unwrap(); cursor += p.consumed; decompressed.extend_from_slice(&dout[..p.produced]); if p.consumed == 0 && p.produced == 0 { break; } } // Final drain in case we have nothing more to feed but staged output remains. loop { let p = ds.decompress(&[], &mut dout).unwrap(); decompressed.extend_from_slice(&dout[..p.produced]); if p.produced == 0 { break; } } assert!(ds.finished(), "decoder did not finalise"); decompressed } #[test] fn pstream_small_roundtrip() { let data = b"Hello pstream! This is a small message that round-trips through the push API."; let out = pstream_roundtrip(data, None, None); assert_eq!(out, data); } #[test] fn pstream_with_checksum() { let data: Vec = (0..32 * 1024).map(|i| (i % 251) as u8).collect(); let copts = CompressOptions { level: Level::Default, checksum: true, seekable: false, }; let dopts = DecompressOptions { verify_checksum: true, }; let out = pstream_roundtrip(&data, Some(&copts), Some(&dopts)); assert_eq!(out, data); } #[test] fn pstream_multi_block() { // Larger than one default block (512 KB) to force multiple blocks. let data: Vec = (0..1536 * 1024).map(|i| ((i * 7) % 256) as u8).collect(); let out = pstream_roundtrip(&data, None, None); assert_eq!(out, data); } #[test] fn pstream_size_hints_nonzero() { let cs = CStream::new(None).unwrap(); assert!(cs.in_size() > 0); assert!(cs.out_size() > 0); let ds = DStream::new(None).unwrap(); assert!(ds.in_size() > 0); assert!(ds.out_size() > 0); assert!(!ds.finished()); } } zxc-0.11.0/wrappers/rust/zxc/src/stdio.rs000066400000000000000000000327361520102567100203320ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ //! [`std::io::Read`] / [`std::io::Write`] adapters over the push streaming API. //! //! Mirror of the wrapper exposed by the Go binding: turns a [`CStream`] / //! [`DStream`] pair into the standard streaming traits so ZXC can be plugged //! into pipelines that expect them. use std::io::{self, Read, Write}; use crate::{CStream, CompressOptions, DStream, DecompressOptions, Error}; /// Magic word identifying a ZXC file frame: little-endian `0x9CB02EF5`. const ZXC_MAGIC_LE: [u8; 4] = [0xF5, 0x2E, 0xB0, 0x9C]; /// Reports whether `data` starts with the ZXC file magic word. /// /// Useful for content-type sniffing in containers / object stores that need /// to decide which decoder to dispatch. /// The check is cheap and side-effect free; it does not validate the rest of /// the header or the footer. /// /// # Example /// /// ```rust /// use zxc::{compress, detect_zxc, Level}; /// /// let frame = compress(b"hello", Level::Default, None).unwrap(); /// assert!(detect_zxc(&frame)); /// assert!(!detect_zxc(b"not a zxc frame")); /// ``` pub fn detect_zxc(data: &[u8]) -> bool { data.len() >= 4 && data[..4] == ZXC_MAGIC_LE } // --------------------------------------------------------------------------- // Encoder // --------------------------------------------------------------------------- /// Streaming compressor implementing [`std::io::Write`]. /// /// Writes are forwarded through a [`CStream`] and the resulting ZXC frame is /// flushed to the inner writer. The frame is finalised (residual block, EOF /// marker, and footer) by [`Encoder::finish`]. [`Drop`] performs a best-effort /// finish but cannot surface errors — prefer [`Encoder::finish`] when error /// handling matters. /// /// `Encoder` is single-threaded; one stream per writer. /// /// # Example /// /// ```rust,no_run /// use std::io::Write; /// use zxc::Encoder; /// /// let sink = Vec::new(); /// let mut enc = Encoder::new(sink).unwrap(); /// enc.write_all(b"hello world").unwrap(); /// let compressed: Vec = enc.finish().unwrap(); /// ``` pub struct Encoder { inner: Option, cs: Option, out_buf: Vec, } impl Encoder { /// Creates an encoder with default compression options. pub fn new(writer: W) -> Result { Self::with_options(writer, None) } /// Creates an encoder honouring `opts`. `opts.seekable` is ignored (push /// API is single-threaded and never emits a seek table). pub fn with_options(writer: W, opts: Option<&CompressOptions>) -> Result { let cs = CStream::new(opts)?; let cap = cs.out_size(); Ok(Self { inner: Some(writer), cs: Some(cs), out_buf: vec![0u8; cap], }) } /// Finalises the frame, drains the inner writer, and returns it. /// /// After this call the encoder is consumed; no further writes are /// possible. pub fn finish(mut self) -> io::Result { self.do_finish()?; // Safety: do_finish leaves both fields populated on success. Ok(self.inner.take().expect("inner writer present")) } /// Returns a reference to the underlying writer. pub fn get_ref(&self) -> &W { self.inner.as_ref().expect("encoder not finished") } /// Returns a mutable reference to the underlying writer. pub fn get_mut(&mut self) -> &mut W { self.inner.as_mut().expect("encoder not finished") } fn do_finish(&mut self) -> io::Result<()> { let (Some(cs), Some(w)) = (self.cs.as_mut(), self.inner.as_mut()) else { return Ok(()); }; loop { let p = cs.end(&mut self.out_buf).map_err(map_err)?; if p.produced > 0 { w.write_all(&self.out_buf[..p.produced])?; } if p.pending == 0 { break; } } // Drop the CStream now so a later finish() / Drop is a no-op. self.cs = None; Ok(()) } } impl Write for Encoder { fn write(&mut self, buf: &[u8]) -> io::Result { if buf.is_empty() { return Ok(0); } let cs = self .cs .as_mut() .ok_or_else(|| io::Error::other("encoder finished"))?; let w = self .inner .as_mut() .ok_or_else(|| io::Error::other("encoder finished"))?; let mut total = 0; let mut input = buf; while !input.is_empty() { let p = cs.compress(input, &mut self.out_buf).map_err(map_err)?; if p.produced > 0 { w.write_all(&self.out_buf[..p.produced])?; } total += p.consumed; input = &input[p.consumed..]; if p.consumed == 0 && p.produced == 0 && p.pending == 0 { // No forward progress is possible — bail out to avoid spin. break; } } Ok(total) } /// Flush is a no-op on the inner writer side: the ZXC frame is only valid /// once finalised, so partial flushes would produce a corrupted frame. /// Use [`Encoder::finish`] to complete the frame. fn flush(&mut self) -> io::Result<()> { if let Some(w) = self.inner.as_mut() { w.flush() } else { Ok(()) } } } impl Drop for Encoder { fn drop(&mut self) { // Best-effort: any error is silently swallowed because Drop cannot // surface failure. Callers who care about errors must call finish(). let _ = self.do_finish(); } } // --------------------------------------------------------------------------- // Decoder // --------------------------------------------------------------------------- /// Streaming decompressor implementing [`std::io::Read`]. /// /// Pulls compressed bytes from the inner reader and yields decompressed /// bytes. Returns [`io::ErrorKind::UnexpectedEof`] if the underlying reader /// is drained before the ZXC footer is reached. /// /// `Decoder` is single-threaded; one stream per reader. /// /// # Example /// /// ```rust,no_run /// use std::io::Read; /// use zxc::Decoder; /// /// let frame: &[u8] = &[]; // a ZXC frame /// let mut dec = Decoder::new(frame).unwrap(); /// let mut buf = Vec::new(); /// dec.read_to_end(&mut buf).unwrap(); /// ``` pub struct Decoder { inner: R, ds: DStream, in_buf: Vec, in_pos: usize, in_len: usize, eof: bool, } impl Decoder { /// Creates a decoder with default decompression options. pub fn new(reader: R) -> Result { Self::with_options(reader, None) } /// Creates a decoder honouring `opts`. pub fn with_options(reader: R, opts: Option<&DecompressOptions>) -> Result { let ds = DStream::new(opts)?; let cap = ds.in_size(); Ok(Self { inner: reader, ds, in_buf: vec![0u8; cap], in_pos: 0, in_len: 0, eof: false, }) } /// Returns a reference to the underlying reader. pub fn get_ref(&self) -> &R { &self.inner } /// Returns a mutable reference to the underlying reader. pub fn get_mut(&mut self) -> &mut R { &mut self.inner } /// Consumes the decoder and returns the inner reader. pub fn into_inner(self) -> R { self.inner } /// Reports whether the decoder has reached and validated the file footer. pub fn finished(&self) -> bool { self.ds.finished() } } impl Read for Decoder { fn read(&mut self, buf: &mut [u8]) -> io::Result { if buf.is_empty() { return Ok(0); } loop { if self.ds.finished() { return Ok(0); } // Try to decompress whatever is currently buffered (or drain mode // when src is at EOF). if self.in_pos < self.in_len || self.eof { let p = self .ds .decompress(&self.in_buf[self.in_pos..self.in_len], buf) .map_err(map_err)?; self.in_pos += p.consumed; if p.produced > 0 { return Ok(p.produced); } if p.consumed == 0 { if self.eof { if self.ds.finished() { return Ok(0); } return Err(io::Error::new( io::ErrorKind::UnexpectedEof, "zxc: input drained before footer", )); } // fall through to refill } else { // consumed > 0 but no output — loop and retry. continue; } } // Refill: shift consumed bytes out, then read more from src. if self.in_pos > 0 { self.in_buf.copy_within(self.in_pos..self.in_len, 0); self.in_len -= self.in_pos; self.in_pos = 0; } let n = self.inner.read(&mut self.in_buf[self.in_len..])?; if n == 0 { self.eof = true; } else { self.in_len += n; } } } } // --------------------------------------------------------------------------- // Error mapping // --------------------------------------------------------------------------- fn map_err(e: Error) -> io::Error { io::Error::other(e) } // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- #[cfg(test)] mod tests { use super::*; use crate::compress; use crate::Level; use std::io::{Cursor, Read, Write}; fn roundtrip(data: &[u8]) -> Vec { let mut enc = Encoder::new(Vec::new()).expect("encoder"); enc.write_all(data).expect("write"); let frame = enc.finish().expect("finish"); let mut dec = Decoder::new(Cursor::new(frame)).expect("decoder"); let mut out = Vec::new(); dec.read_to_end(&mut out).expect("read_to_end"); out } #[test] fn encoder_decoder_small_roundtrip() { let data = b"Hello std::io::Read / std::io::Write bridge over ZXC."; assert_eq!(roundtrip(data), data); } #[test] fn encoder_decoder_large_roundtrip() { let data: Vec = (0..2 * 1024 * 1024).map(|i| ((i * 13) % 251) as u8).collect(); assert_eq!(roundtrip(&data), data); } #[test] fn encoder_many_small_writes() { let mut enc = Encoder::new(Vec::new()).unwrap(); let mut want: Vec = Vec::with_capacity(64 * 1024); for i in 0u32..4096 { let chunk = [i as u8, (i >> 8) as u8, (i ^ 0x5A) as u8]; want.extend_from_slice(&chunk); enc.write_all(&chunk).unwrap(); } let frame = enc.finish().unwrap(); let mut dec = Decoder::new(Cursor::new(frame)).unwrap(); let mut got = Vec::new(); dec.read_to_end(&mut got).unwrap(); assert_eq!(got, want); } #[test] fn encoder_decoder_with_checksum() { use crate::{CompressOptions, DecompressOptions}; let data: Vec = (0..32 * 1024).map(|i| i as u8).collect(); let copts = CompressOptions { checksum: true, ..Default::default() }; let dopts = DecompressOptions { verify_checksum: true, }; let mut enc = Encoder::with_options(Vec::new(), Some(&copts)).unwrap(); enc.write_all(&data).unwrap(); let frame = enc.finish().unwrap(); let mut dec = Decoder::with_options(Cursor::new(frame), Some(&dopts)).unwrap(); let mut got = Vec::new(); dec.read_to_end(&mut got).unwrap(); assert_eq!(got, data); } #[test] fn drop_finishes_frame() { // Encoder is dropped without an explicit finish() — the resulting // frame must still be well-formed (best-effort flush in Drop). let mut sink: Vec = Vec::new(); { let mut enc = Encoder::new(&mut sink).unwrap(); enc.write_all(b"drop-flush").unwrap(); } let mut dec = Decoder::new(Cursor::new(sink)).unwrap(); let mut got = Vec::new(); dec.read_to_end(&mut got).unwrap(); assert_eq!(got, b"drop-flush"); } #[test] fn decoder_truncated_frame_errors() { let frame = compress(&vec![b'A'; 32 * 1024], Level::Default, None).unwrap(); let truncated = &frame[..frame.len() / 2]; let mut dec = Decoder::new(Cursor::new(truncated)).unwrap(); let mut got = Vec::new(); let err = dec.read_to_end(&mut got).unwrap_err(); assert_eq!(err.kind(), io::ErrorKind::UnexpectedEof); } #[test] fn detect_zxc_basic() { let frame = compress(b"sniff me", Level::Default, None).unwrap(); assert!(detect_zxc(&frame)); let mut enc = Encoder::new(Vec::new()).unwrap(); enc.write_all(b"hi").unwrap(); let frame = enc.finish().unwrap(); assert!(detect_zxc(&frame)); assert!(!detect_zxc(&[])); assert!(!detect_zxc(&[0xF5, 0x2E, 0xB0])); // 3 bytes, too short assert!(!detect_zxc(&[0; 4])); assert!(!detect_zxc(b"not a zxc frame at all")); } } zxc-0.11.0/wrappers/wasm/000077500000000000000000000000001520102567100152065ustar00rootroot00000000000000zxc-0.11.0/wrappers/wasm/.nvmrc000066400000000000000000000000021520102567100163240ustar00rootroot0000000000000022zxc-0.11.0/wrappers/wasm/README.md000066400000000000000000000061721520102567100164730ustar00rootroot00000000000000# ZXC WebAssembly High-performance lossless compression for the browser and Node.js via WebAssembly. ## Features - **Buffer API**: Compress and decompress `Uint8Array` buffers - **Reusable Contexts**: Amortise allocation overhead across multiple operations - **All Levels**: Compression levels 1–5 - **Checksum Support**: Optional integrity verification - **Tiny Footprint**: ~60 KB `.wasm` file (scalar build, no SIMD) ## Quick Start ### Browser (ES Module) ```js import createZXC from './zxc_wasm.js'; const zxc = await createZXC(); // Compress const input = new TextEncoder().encode('Hello, World!'); const compressed = zxc.compress(input, { level: 3 }); // Decompress const output = zxc.decompress(compressed); console.log(new TextDecoder().decode(output)); // "Hello, World!" ``` ### Node.js ```js import createZXC from 'zxc-wasm'; import { readFileSync } from 'fs'; const zxc = await createZXC(); const data = new Uint8Array(readFileSync('input.bin')); const compressed = zxc.compress(data, { level: 5, checksum: true }); const decompressed = zxc.decompress(compressed, { checksum: true }); ``` ## API Reference ### `createZXC(moduleOverrides?) -> Promise` Initialise the WASM module. Returns a frozen API object. ### `zxc.compress(data, opts?) -> Uint8Array` | Option | Type | Default | Description | |--------|------|---------|-------------| | `level` | `number` | `3` | Compression level (1–5) | | `checksum` | `boolean` | `false` | Enable integrity checksums | ### `zxc.decompress(data, opts?) -> Uint8Array` | Option | Type | Default | Description | |--------|------|---------|-------------| | `checksum` | `boolean` | `false` | Verify integrity checksums | ### `zxc.compressBound(inputSize) -> number` Maximum compressed output size for a given input size. ### `zxc.getDecompressedSize(data) -> number` Read the original size from a compressed buffer without decompressing. ### `zxc.createCompressContext(opts?) -> CompressContext` Create a reusable compression context (avoids per-call allocation). ```js const ctx = zxc.createCompressContext({ level: 3 }); const c1 = ctx.compress(data1); const c2 = ctx.compress(data2); ctx.free(); // Release WASM memory ``` ### `zxc.createDecompressContext() -> DecompressContext` Create a reusable decompression context. ```js const ctx = zxc.createDecompressContext(); const d1 = ctx.decompress(compressed1); const d2 = ctx.decompress(compressed2); ctx.free(); ``` ### Properties | Property | Type | Description | |----------|------|-------------| | `version` | `string` | Library version (e.g. `"0.11.0"`) | | `minLevel` | `number` | Minimum compression level (`1`) | | `maxLevel` | `number` | Maximum compression level (`5`) | | `defaultLevel` | `number` | Default compression level (`3`) | ## Building from Source Requires [Emscripten SDK](https://emscripten.org/docs/getting_started/downloads.html). ```bash # Configure emcmake cmake -B build-wasm -DCMAKE_BUILD_TYPE=Release # Build cmake --build build-wasm # Test BUILD_DIR=build-wasm node wrappers/wasm/test.mjs ``` The build produces `build-wasm/zxc.js` and `build-wasm/zxc.wasm`. ## License BSD 3-Clause. See [LICENSE](../../LICENSE). zxc-0.11.0/wrappers/wasm/package.json000066400000000000000000000012711520102567100174750ustar00rootroot00000000000000{ "name": "zxc-wasm", "version": "0.11.0", "description": "ZXC high-performance lossless compression - WebAssembly build", "type": "module", "main": "zxc_wasm.js", "exports": { ".": "./zxc_wasm.js" }, "files": [ "zxc_wasm.js", "zxc.js", "zxc.wasm", "README.md" ], "scripts": { "test": "node test.mjs" }, "keywords": [ "compression", "decompression", "lz77", "wasm", "webassembly", "zxc", "lossless", "high-performance" ], "author": "Bertrand Lebonnois", "license": "BSD-3-Clause", "repository": { "type": "git", "url": "https://github.com/hellobertrand/zxc.git", "directory": "wrappers/wasm" } }zxc-0.11.0/wrappers/wasm/test.mjs000066400000000000000000000360711520102567100167070ustar00rootroot00000000000000/** * ZXC WASM Roundtrip Test * * Validates compress -> decompress correctness via Node.js. * Run: node wrappers/wasm/test.mjs * * Expects the built zxc.js + zxc.wasm to be in the build directory. * The BUILD_DIR environment variable can override the default path. * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ import { createRequire } from 'module'; import { join, dirname, resolve } from 'path'; import { fileURLToPath } from 'url'; const __dirname = dirname(fileURLToPath(import.meta.url)); // Resolve BUILD_DIR relative to CWD (not the test file location) const buildDir = process.env.BUILD_DIR ? resolve(process.cwd(), process.env.BUILD_DIR) : join(__dirname, '..', '..', 'build-wasm'); // Emscripten generates a CJS module; use createRequire to load it. const require = createRequire(import.meta.url); const ZXCModule = require(join(buildDir, 'zxc.js')); let passed = 0; let failed = 0; function assert(cond, msg) { if (!cond) { console.error(` ✗ FAIL: ${msg}`); failed++; } else { console.log(` ✓ ${msg}`); passed++; } } function arraysEqual(a, b) { if (a.length !== b.length) return false; for (let i = 0; i < a.length; i++) { if (a[i] !== b[i]) return false; } return true; } async function main() { console.log('ZXC WASM Test Suite\n'); // --- 1. Module initialisation --- console.log('1. Module Initialisation'); const Module = await ZXCModule(); assert(!!Module, 'Module loaded successfully'); // Wrap core functions const _compress = Module.cwrap('zxc_compress', 'number', ['number', 'number', 'number', 'number', 'number']); const _decompress = Module.cwrap('zxc_decompress', 'number', ['number', 'number', 'number', 'number', 'number']); const _compress_bound = Module.cwrap('zxc_compress_bound', 'number', ['number']); const _get_decompressed_size = Module.cwrap('zxc_get_decompressed_size', 'number', ['number', 'number']); const _version_string = Module.cwrap('zxc_version_string', 'string', []); const _min_level = Module.cwrap('zxc_min_level', 'number', []); const _max_level = Module.cwrap('zxc_max_level', 'number', []); const _default_level = Module.cwrap('zxc_default_level', 'number', []); const _error_name = Module.cwrap('zxc_error_name', 'string', ['number']); // --- 2. Library info --- console.log('\n2. Library Info'); const version = _version_string(); assert(typeof version === 'string' && version.length > 0, `Version: ${version}`); assert(_min_level() === 1, `Min level: ${_min_level()}`); assert(_max_level() === 6, `Max level: ${_max_level()}`); assert(_default_level() === 3, `Default level: ${_default_level()}`); // --- 3. Compress bound --- console.log('\n3. Compress Bound'); const bound1k = _compress_bound(1024); assert(bound1k > 1024, `Bound for 1KB: ${bound1k}`); const bound0 = _compress_bound(0); assert(bound0 > 0, `Bound for 0 bytes: ${bound0}`); // --- 4. Roundtrip: simple string --- console.log('\n4. Roundtrip: Simple String'); { const input = new TextEncoder().encode('Hello, ZXC WebAssembly! '.repeat(100)); const bound = _compress_bound(input.length); const srcPtr = Module._malloc(input.length); const dstPtr = Module._malloc(bound); Module.HEAPU8.set(input, srcPtr); const csize = _compress(srcPtr, input.length, dstPtr, bound, 0); assert(csize > 0, `Compressed ${input.length} -> ${csize} bytes`); assert(csize < input.length, `Compression ratio: ${(input.length / csize).toFixed(2)}x`); // Read back compressed data const compressed = new Uint8Array(Module.HEAPU8.buffer, dstPtr, csize).slice(); // Get decompressed size const csrcPtr = Module._malloc(compressed.length); Module.HEAPU8.set(compressed, csrcPtr); const origSize = _get_decompressed_size(csrcPtr, compressed.length); assert(origSize === input.length, `Decompressed size: ${origSize} (expected ${input.length})`); // Decompress const outPtr = Module._malloc(origSize); const dsize = _decompress(csrcPtr, compressed.length, outPtr, origSize, 0); assert(dsize === input.length, `Decompressed ${dsize} bytes`); const output = new Uint8Array(Module.HEAPU8.buffer, outPtr, dsize).slice(); assert(arraysEqual(input, output), 'Roundtrip byte-exact match'); Module._free(srcPtr); Module._free(dstPtr); Module._free(csrcPtr); Module._free(outPtr); } // --- 5. Roundtrip: all compression levels --- console.log('\n5. Roundtrip: All Compression Levels'); { const input = new Uint8Array(4096); // Fill with semi-compressible pattern for (let i = 0; i < input.length; i++) { input[i] = (i * 7 + 13) & 0xFF; } const bound = _compress_bound(input.length); for (let level = 1; level <= 6; level++) { const srcPtr = Module._malloc(input.length); const dstPtr = Module._malloc(bound); Module.HEAPU8.set(input, srcPtr); // Build opts struct: {n_threads=0, level, block_size=0, checksum=1, seekable=0, ...} const optsPtr = Module._malloc(28); for (let i = 0; i < 28; i++) Module.HEAPU8[optsPtr + i] = 0; Module.HEAP32[(optsPtr >> 2) + 1] = level; // level Module.HEAP32[(optsPtr >> 2) + 3] = 1; // checksum_enabled const csize = _compress(srcPtr, input.length, dstPtr, bound, optsPtr); assert(csize > 0, `Level ${level}: compressed to ${csize} bytes`); // Decompress with checksum verification const compressed = new Uint8Array(Module.HEAPU8.buffer, dstPtr, csize).slice(); const csrcPtr = Module._malloc(compressed.length); Module.HEAPU8.set(compressed, csrcPtr); const dOptsPtr = Module._malloc(16); for (let i = 0; i < 16; i++) Module.HEAPU8[dOptsPtr + i] = 0; Module.HEAP32[(dOptsPtr >> 2) + 1] = 1; // checksum_enabled const outPtr = Module._malloc(input.length); const dsize = _decompress(csrcPtr, compressed.length, outPtr, input.length, dOptsPtr); const output = new Uint8Array(Module.HEAPU8.buffer, outPtr, dsize).slice(); assert(arraysEqual(input, output), `Level ${level}: roundtrip OK`); Module._free(srcPtr); Module._free(dstPtr); Module._free(optsPtr); Module._free(csrcPtr); Module._free(dOptsPtr); Module._free(outPtr); } } // --- 6. Reusable context API --- console.log('\n6. Reusable Context API'); { const _create_cctx = Module.cwrap('zxc_create_cctx', 'number', ['number']); const _free_cctx = Module.cwrap('zxc_free_cctx', 'void', ['number']); const _compress_cctx = Module.cwrap('zxc_compress_cctx', 'number', ['number', 'number', 'number', 'number', 'number', 'number']); const _create_dctx = Module.cwrap('zxc_create_dctx', 'number', []); const _free_dctx = Module.cwrap('zxc_free_dctx', 'void', ['number']); const _decompress_dctx = Module.cwrap('zxc_decompress_dctx', 'number', ['number', 'number', 'number', 'number', 'number', 'number']); const cctx = _create_cctx(0); assert(cctx !== 0, 'Created compression context'); const dctx = _create_dctx(); assert(dctx !== 0, 'Created decompression context'); // Compress two different payloads with same context for (let trial = 0; trial < 3; trial++) { const input = new TextEncoder().encode(`Trial ${trial}: ${'ABCDEFGH'.repeat(200)}`); const bound = _compress_bound(input.length); const srcPtr = Module._malloc(input.length); const dstPtr = Module._malloc(bound); Module.HEAPU8.set(input, srcPtr); const csize = _compress_cctx(cctx, srcPtr, input.length, dstPtr, bound, 0); assert(csize > 0, `Context compress trial ${trial}: ${csize} bytes`); const compressed = new Uint8Array(Module.HEAPU8.buffer, dstPtr, csize).slice(); const csrcPtr = Module._malloc(compressed.length); Module.HEAPU8.set(compressed, csrcPtr); const outPtr = Module._malloc(input.length); const dsize = _decompress_dctx(dctx, csrcPtr, compressed.length, outPtr, input.length, 0); const output = new Uint8Array(Module.HEAPU8.buffer, outPtr, dsize).slice(); assert(arraysEqual(input, output), `Context roundtrip trial ${trial}: OK`); Module._free(srcPtr); Module._free(dstPtr); Module._free(csrcPtr); Module._free(outPtr); } _free_cctx(cctx); _free_dctx(dctx); console.log(' Contexts freed.'); } // --- 7. Error handling --- console.log('\n7. Error Handling'); { const errorName = _error_name(-1); assert(typeof errorName === 'string', `Error name for -1: ${errorName}`); // Attempt to decompress garbage const garbage = new Uint8Array([0, 1, 2, 3, 4, 5, 6, 7]); const gPtr = Module._malloc(garbage.length); const outPtr = Module._malloc(1024); Module.HEAPU8.set(garbage, gPtr); const result = _decompress(gPtr, garbage.length, outPtr, 1024, 0); assert(result < 0, `Garbage decompression returns error: ${result} (${_error_name(result)})`); Module._free(gPtr); Module._free(outPtr); } // --- 8. Push streaming API (high-level wrapper) ----------------------- console.log('\n8. Push Streaming API (CStream / DStream)'); { const { default: createZXC } = await import('./zxc_wasm.js'); const zxc = await createZXC({}, ZXCModule); const cs = zxc.createCStream({ checksum: true }); assert(cs.inSize() > 0 && cs.outSize() > 0, `cstream size hints: in=${cs.inSize()} out=${cs.outSize()}`); // Multi-block payload to exercise > 1 block. const data = new Uint8Array(512 * 1024); for (let i = 0; i < data.length; i++) data[i] = (i * 7) & 0xff; const chunks = []; let totalC = 0; const step = 17_000; for (let i = 0; i < data.length; i += step) { const c = cs.compress(data.subarray(i, Math.min(i + step, data.length))); chunks.push(c); totalC += c.length; } const tail = cs.end(); chunks.push(tail); totalC += tail.length; cs.free(); const compressed = new Uint8Array(totalC); let off = 0; for (const c of chunks) { compressed.set(c, off); off += c.length; } assert(compressed.length > 0, `pstream compressed ${data.length} -> ${compressed.length} bytes`); const ds = zxc.createDStream({ checksum: true }); const outChunks = []; let totalD = 0; const dstep = 31_000; for (let i = 0; i < compressed.length; i += dstep) { const o = ds.decompress(compressed.subarray(i, Math.min(i + dstep, compressed.length))); outChunks.push(o); totalD += o.length; } assert(ds.finished(), 'dstream finished after full input'); ds.free(); const decoded = new Uint8Array(totalD); off = 0; for (const c of outChunks) { decoded.set(c, off); off += c.length; } assert(decoded.length === data.length, `pstream decoded ${decoded.length} bytes (expected ${data.length})`); assert(arraysEqual(data, decoded), 'pstream roundtrip byte-exact match'); } // --- 9. WHATWG TransformStream adapters ------------------------------- console.log('\n9. WHATWG TransformStream adapters + detectZxc'); { const { default: createZXC, detectZxc } = await import('./zxc_wasm.js'); const zxc = await createZXC({}, ZXCModule); // detectZxc: works without WASM init too (pure JS), but check both // call sites as they share the same implementation. assert(detectZxc === zxc.detectZxc, 'detectZxc exported at module + API level'); assert(!detectZxc(null), 'detectZxc(null) === false'); assert(!detectZxc(new Uint8Array([0xF5, 0x2E, 0xB0])), 'detectZxc rejects 3-byte input'); assert(!detectZxc(new Uint8Array(4)), 'detectZxc rejects zero buffer'); // Build a frame with the buffer API and sniff it. const frame = zxc.compress(new Uint8Array([1, 2, 3, 4, 5])); assert(detectZxc(frame), 'detectZxc accepts compress() output'); assert(detectZxc(frame.buffer), 'detectZxc accepts ArrayBuffer'); // TransformStream roundtrip via pipeThrough(). const data = new Uint8Array(64 * 1024); for (let i = 0; i < data.length; i++) data[i] = (i ^ 0x5A) & 0xFF; const sourceCompressed = new ReadableStream({ start(c) { c.enqueue(data); c.close(); } }).pipeThrough(zxc.createCompressTransformStream()); const compressedChunks = []; let cTotal = 0; for await (const c of sourceCompressed) { compressedChunks.push(c); cTotal += c.length; } const compressed = new Uint8Array(cTotal); { let off = 0; for (const c of compressedChunks) { compressed.set(c, off); off += c.length; } } assert(detectZxc(compressed), 'TransformStream output is a valid ZXC frame'); const sourceDecompressed = new ReadableStream({ start(c) { c.enqueue(compressed); c.close(); } }).pipeThrough(zxc.createDecompressTransformStream()); const outChunks = []; let dTotal = 0; for await (const c of sourceDecompressed) { outChunks.push(c); dTotal += c.length; } const decoded = new Uint8Array(dTotal); { let off = 0; for (const c of outChunks) { decoded.set(c, off); off += c.length; } } assert(decoded.length === data.length, `TransformStream decoded ${decoded.length} bytes (expected ${data.length})`); assert(arraysEqual(data, decoded), 'TransformStream roundtrip byte-exact match'); // Truncated frame must error the decode stream. const truncated = compressed.subarray(0, Math.floor(compressed.length / 2)); const truncSource = new ReadableStream({ start(c) { c.enqueue(truncated); c.close(); } }).pipeThrough(zxc.createDecompressTransformStream()); let didThrow = false; let errorCode; try { for await (const _ of truncSource) { /* drain */ } } catch (e) { didThrow = true; errorCode = e?.code; } assert(didThrow, 'truncated frame throws an error'); assert(errorCode === 'ZXC_TRUNCATED', 'truncated frame error code is ZXC_TRUNCATED'); } // --- Summary --- console.log(`\n${'='.repeat(40)}`); console.log(`Results: ${passed} passed, ${failed} failed`); process.exit(failed > 0 ? 1 : 0); } main().catch(err => { console.error('Fatal error:', err); process.exit(1); }); zxc-0.11.0/wrappers/wasm/wasm_entry.c000066400000000000000000000010261520102567100175410ustar00rootroot00000000000000/* * ZXC - High-performance lossless compression * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /** * @file wasm_entry.c * @brief Minimal entry point for the WebAssembly build. * * Emscripten requires a main() symbol when producing a .js + .wasm pair * via add_executable(). All useful functionality is exported directly * from the library through -sEXPORTED_FUNCTIONS; this file simply * provides the mandatory entry point. */ int main(void) { return 0; } zxc-0.11.0/wrappers/wasm/zxc_wasm.js000066400000000000000000000616711520102567100174120ustar00rootroot00000000000000/** * ZXC WebAssembly Wrapper * * High-level JavaScript API for ZXC compression/decompression via WASM. * * Copyright (c) 2025-2026 Bertrand Lebonnois and contributors. * SPDX-License-Identifier: BSD-3-Clause */ /** * Returns true if `buf` starts with the ZXC file magic word * (little-endian 0x9CB02EF5). * * Side-effect free, synchronous, does not require WASM initialisation — * suitable for content-type sniffing in fetch/Service Worker pipelines and * for OCI media-type negotiation. * * @param {Uint8Array | ArrayBuffer | null | undefined} buf * @returns {boolean} */ export function detectZxc(buf) { if (!buf) return false; const view = buf instanceof ArrayBuffer ? new Uint8Array(buf) : buf; if (!view || view.length < 4) return false; return view[0] === 0xF5 && view[1] === 0x2E && view[2] === 0xB0 && view[3] === 0x9C; } /** * Initialise the ZXC WASM module and return an ergonomic API object. * * @param {object} [moduleOverrides] - Optional Emscripten module overrides * (e.g. { locateFile: ... } for custom .wasm path resolution). * @param {Function} [factory] - Optional Emscripten module factory. If * omitted, the default sibling `./zxc.js` is dynamically imported. * @returns {Promise} Resolved API object. */ export default async function createZXC(moduleOverrides, factory) { const ZXCModule = factory || (await import('./zxc.js')).default; const Module = await ZXCModule(moduleOverrides || {}); // --- Wrap C functions via cwrap ------------------------------------------ const _compress = Module.cwrap('zxc_compress', 'number', ['number', 'number', 'number', 'number', 'number']); const _decompress = Module.cwrap('zxc_decompress', 'number', ['number', 'number', 'number', 'number', 'number']); const _compress_bound = Module.cwrap('zxc_compress_bound', 'number', ['number']); const _get_decompressed_size = Module.cwrap('zxc_get_decompressed_size', 'number', ['number', 'number']); const _create_cctx = Module.cwrap('zxc_create_cctx', 'number', ['number']); const _free_cctx = Module.cwrap('zxc_free_cctx', 'void', ['number']); const _compress_cctx = Module.cwrap('zxc_compress_cctx', 'number', ['number', 'number', 'number', 'number', 'number', 'number']); const _create_dctx = Module.cwrap('zxc_create_dctx', 'number', []); const _free_dctx = Module.cwrap('zxc_free_dctx', 'void', ['number']); const _decompress_dctx = Module.cwrap('zxc_decompress_dctx', 'number', ['number', 'number', 'number', 'number', 'number', 'number']); // Push streaming API const _cstream_create = Module.cwrap('zxc_cstream_create', 'number', ['number']); const _cstream_free = Module.cwrap('zxc_cstream_free', 'void', ['number']); const _cstream_compress = Module.cwrap('zxc_cstream_compress', 'number', ['number', 'number', 'number']); const _cstream_end = Module.cwrap('zxc_cstream_end', 'number', ['number', 'number']); const _cstream_in_size = Module.cwrap('zxc_cstream_in_size', 'number', ['number']); const _cstream_out_size = Module.cwrap('zxc_cstream_out_size', 'number', ['number']); const _dstream_create = Module.cwrap('zxc_dstream_create', 'number', ['number']); const _dstream_free = Module.cwrap('zxc_dstream_free', 'void', ['number']); const _dstream_decompress = Module.cwrap('zxc_dstream_decompress', 'number', ['number', 'number', 'number']); const _dstream_finished = Module.cwrap('zxc_dstream_finished', 'number', ['number']); const _dstream_in_size = Module.cwrap('zxc_dstream_in_size', 'number', ['number']); const _dstream_out_size = Module.cwrap('zxc_dstream_out_size', 'number', ['number']); const _version_string = Module.cwrap('zxc_version_string', 'string', []); const _error_name = Module.cwrap('zxc_error_name', 'string', ['number']); const _min_level = Module.cwrap('zxc_min_level', 'number', []); const _max_level = Module.cwrap('zxc_max_level', 'number', []); const _default_level = Module.cwrap('zxc_default_level', 'number', []); const _malloc = Module._malloc; const _free = Module._free; // --- Options struct layout ----------------------------------------------- // zxc_compress_opts_t (WASM32 layout): // int n_threads (4) | int level (4) | size_t block_size (4 in wasm32) // int checksum_enabled (4) | int seekable (4) // ptr progress_cb (4) | ptr user_data (4) // Total: 28 bytes in WASM32 const COMPRESS_OPTS_SIZE = 28; // zxc_decompress_opts_t: // int n_threads (4) | int checksum_enabled (4) | ptr progress_cb (4) | ptr user_data (4) // Total: 16 bytes in WASM32 const DECOMPRESS_OPTS_SIZE = 16; // zxc_inbuf_t / zxc_outbuf_t (WASM32 layout): // ptr src/dst (4) | size_t size (4) | size_t pos (4) // Total: 12 bytes in WASM32 const IO_BUF_SIZE = 12; /** * Write a zxc_compress_opts_t struct into WASM memory. * @returns {number} Pointer to the struct (caller must free). */ function _writeCompressOpts(level, checksum, seekable) { const ptr = _malloc(COMPRESS_OPTS_SIZE); // Zero-fill first for (let i = 0; i < COMPRESS_OPTS_SIZE; i++) { Module.HEAPU8[ptr + i] = 0; } // n_threads = 0 (offset 0) Module.HEAP32[(ptr >> 2) + 0] = 0; // level (offset 4) Module.HEAP32[(ptr >> 2) + 1] = level; // block_size = 0 (default, offset 8) Module.HEAPU32[(ptr >> 2) + 2] = 0; // checksum_enabled (offset 12) Module.HEAP32[(ptr >> 2) + 3] = checksum ? 1 : 0; // seekable (offset 16) Module.HEAP32[(ptr >> 2) + 4] = seekable ? 1 : 0; // progress_cb = NULL (offset 20), user_data = NULL (offset 24) return ptr; } /** * Write a zxc_decompress_opts_t struct into WASM memory. * @returns {number} Pointer to the struct (caller must free). */ function _writeDecompressOpts(checksum) { const ptr = _malloc(DECOMPRESS_OPTS_SIZE); for (let i = 0; i < DECOMPRESS_OPTS_SIZE; i++) { Module.HEAPU8[ptr + i] = 0; } // n_threads = 0 (offset 0) Module.HEAP32[(ptr >> 2) + 0] = 0; // checksum_enabled (offset 4) Module.HEAP32[(ptr >> 2) + 1] = checksum ? 1 : 0; // progress_cb = NULL (offset 8), user_data = NULL (offset 12) return ptr; } // --- Public API ---------------------------------------------------------- /** * Compress a Uint8Array. * * @param {Uint8Array} data - Input data to compress. * @param {object} [opts] - Options. * @param {number} [opts.level=3] - Compression level (1-6). * @param {boolean} [opts.checksum=false] - Enable checksums. * @param {boolean} [opts.seekable=false] - Append seek table for random-access. * @returns {Uint8Array} Compressed data. * @throws {Error} On compression failure. */ function compress(data, opts) { const level = (opts && opts.level) || _default_level(); const checksum = (opts && opts.checksum) || false; const seekable = (opts && opts.seekable) || false; const bound = _compress_bound(data.length); if (bound === 0) throw new Error('ZXC: compress_bound returned 0'); const srcPtr = _malloc(data.length); const dstPtr = _malloc(bound); const optsPtr = _writeCompressOpts(level, checksum, seekable); try { Module.HEAPU8.set(data, srcPtr); const result = _compress(srcPtr, data.length, dstPtr, bound, optsPtr); if (result < 0) { throw new Error(`ZXC compress error: ${_error_name(result)} (${result})`); } return new Uint8Array(Module.HEAPU8.buffer, dstPtr, result).slice(); } finally { _free(srcPtr); _free(dstPtr); _free(optsPtr); } } /** * Decompress a Uint8Array. * * @param {Uint8Array} data - Compressed data. * @param {object} [opts] - Options. * @param {boolean} [opts.checksum=false] - Verify checksums. * @returns {Uint8Array} Decompressed data. * @throws {Error} On decompression failure. */ function decompress(data, opts) { const checksum = (opts && opts.checksum) || false; // Read decompressed size from footer const srcPtr = _malloc(data.length); Module.HEAPU8.set(data, srcPtr); const origSize = _get_decompressed_size(srcPtr, data.length); if (origSize === 0) { _free(srcPtr); throw new Error('ZXC: cannot read decompressed size (invalid archive?)'); } const dstPtr = _malloc(origSize); const optsPtr = _writeDecompressOpts(checksum); try { const result = _decompress(srcPtr, data.length, dstPtr, origSize, optsPtr); if (result < 0) { throw new Error(`ZXC decompress error: ${_error_name(result)} (${result})`); } return new Uint8Array(Module.HEAPU8.buffer, dstPtr, result).slice(); } finally { _free(srcPtr); _free(dstPtr); _free(optsPtr); } } /** * Returns the maximum compressed output size for a given input size. * @param {number} inputSize * @returns {number} */ function compressBound(inputSize) { return _compress_bound(inputSize); } /** * Reads the decompressed size from a compressed buffer without decompressing. * @param {Uint8Array} data - Compressed data. * @returns {number} Original uncompressed size, or 0 if invalid. */ function getDecompressedSize(data) { const ptr = _malloc(data.length); Module.HEAPU8.set(data, ptr); const size = _get_decompressed_size(ptr, data.length); _free(ptr); return size; } /** * Create a reusable compression context for high-frequency usage. * Call .free() when done to release WASM memory. * * @param {object} [opts] - Default options. * @param {number} [opts.level=3] - Default compression level. * @param {boolean} [opts.checksum=false] - Default checksum setting. * @returns {{ compress: Function, free: Function }} */ function createCompressContext(opts) { const level = (opts && opts.level) || _default_level(); const checksum = (opts && opts.checksum) || false; const seekable = (opts && opts.seekable) || false; const optsPtr = _writeCompressOpts(level, checksum, seekable); const cctx = _create_cctx(optsPtr); _free(optsPtr); if (cctx === 0) throw new Error('ZXC: failed to create compression context'); return { /** * Compress data using this reusable context. * @param {Uint8Array} data * @returns {Uint8Array} */ compress(data) { const bound = _compress_bound(data.length); const srcPtr = _malloc(data.length); const dstPtr = _malloc(bound); try { Module.HEAPU8.set(data, srcPtr); const result = _compress_cctx(cctx, srcPtr, data.length, dstPtr, bound, 0); if (result < 0) { throw new Error(`ZXC cctx compress error: ${_error_name(result)} (${result})`); } return new Uint8Array(Module.HEAPU8.buffer, dstPtr, result).slice(); } finally { _free(srcPtr); _free(dstPtr); } }, /** Free the context and release WASM memory. */ free() { _free_cctx(cctx); } }; } /** * Create a reusable decompression context for high-frequency usage. * Call .free() when done to release WASM memory. * * @returns {{ decompress: Function, free: Function }} */ function createDecompressContext() { const dctx = _create_dctx(); if (dctx === 0) throw new Error('ZXC: failed to create decompression context'); return { /** * Decompress data using this reusable context. * @param {Uint8Array} data * @returns {Uint8Array} */ decompress(data) { const srcPtr = _malloc(data.length); Module.HEAPU8.set(data, srcPtr); const origSize = _get_decompressed_size(srcPtr, data.length); if (origSize === 0) { _free(srcPtr); throw new Error('ZXC: cannot read decompressed size'); } const dstPtr = _malloc(origSize); try { const result = _decompress_dctx(dctx, srcPtr, data.length, dstPtr, origSize, 0); if (result < 0) { throw new Error(`ZXC dctx decompress error: ${_error_name(result)} (${result})`); } return new Uint8Array(Module.HEAPU8.buffer, dstPtr, result).slice(); } finally { _free(srcPtr); _free(dstPtr); } }, /** Free the context and release WASM memory. */ free() { _free_dctx(dctx); } }; } // --- Push streaming helpers ---------------------------------------------- /** Write a zxc_inbuf_t at `bufPtr` pointing to `srcPtr` (size bytes, pos=0). */ function _writeInbuf(bufPtr, srcPtr, size) { Module.HEAP32[(bufPtr >> 2) + 0] = srcPtr; Module.HEAPU32[(bufPtr >> 2) + 1] = size; Module.HEAPU32[(bufPtr >> 2) + 2] = 0; } /** Write a zxc_outbuf_t at `bufPtr` pointing to `dstPtr` (size bytes, pos=0). */ function _writeOutbuf(bufPtr, dstPtr, size) { Module.HEAP32[(bufPtr >> 2) + 0] = dstPtr; Module.HEAPU32[(bufPtr >> 2) + 1] = size; Module.HEAPU32[(bufPtr >> 2) + 2] = 0; } function _readPos(bufPtr) { return Module.HEAPU32[(bufPtr >> 2) + 2]; } /* Concatenate JS-side slices into a single Uint8Array. */ function _concatChunks(chunks, totalLen) { const out = new Uint8Array(totalLen); let off = 0; for (const c of chunks) { out.set(c, off); off += c.length; } return out; } /** * Create a push-based, single-threaded compression stream. * @param {object} [opts] * @param {number} [opts.level=3] * @param {boolean} [opts.checksum=false] * @returns {{ compress(Uint8Array): Uint8Array, end(): Uint8Array, free(): void, inSize(): number, outSize(): number }} */ function createCStream(opts) { const level = (opts && opts.level) || _default_level(); const checksum = (opts && opts.checksum) || false; const optsPtr = _writeCompressOpts(level, checksum, false); const cs = _cstream_create(optsPtr); _free(optsPtr); if (cs === 0) throw new Error('ZXC: failed to create cstream'); // Reusable scratch buffers in the WASM heap. The compress/end calls // never reallocate, so these pointers stay valid for the stream's // lifetime even if the heap grows (cwrap re-reads HEAPU8 internally). const inDescPtr = _malloc(IO_BUF_SIZE); const outDescPtr = _malloc(IO_BUF_SIZE); const stageCap = Math.max(_cstream_out_size(cs), 64 * 1024); const stagePtr = _malloc(stageCap); function drainCompress(srcPtr, srcLen) { const chunks = []; let total = 0; _writeInbuf(inDescPtr, srcPtr, srcLen); for (;;) { _writeOutbuf(outDescPtr, stagePtr, stageCap); const r = _cstream_compress(cs, outDescPtr, inDescPtr); if (r < 0) { throw new Error(`ZXC cstream compress error: ${_error_name(r)} (${r})`); } const produced = _readPos(outDescPtr); if (produced > 0) { chunks.push(new Uint8Array(Module.HEAPU8.buffer, stagePtr, produced).slice()); total += produced; } if (r === 0 && _readPos(inDescPtr) === srcLen) break; } return _concatChunks(chunks, total); } function drainEnd() { const chunks = []; let total = 0; for (;;) { _writeOutbuf(outDescPtr, stagePtr, stageCap); const r = _cstream_end(cs, outDescPtr); if (r < 0) { throw new Error(`ZXC cstream end error: ${_error_name(r)} (${r})`); } const produced = _readPos(outDescPtr); if (produced > 0) { chunks.push(new Uint8Array(Module.HEAPU8.buffer, stagePtr, produced).slice()); total += produced; } if (r === 0) break; } return _concatChunks(chunks, total); } return { /** Push input and return any compressed bytes produced. */ compress(data) { if (data.length === 0) return drainCompress(stagePtr, 0); const srcPtr = _malloc(data.length); try { Module.HEAPU8.set(data, srcPtr); return drainCompress(srcPtr, data.length); } finally { _free(srcPtr); } }, /** Finalise: residual block + EOF + footer. */ end() { return drainEnd(); }, /** Free the stream and its scratch buffers. */ free() { _cstream_free(cs); _free(inDescPtr); _free(outDescPtr); _free(stagePtr); }, inSize() { return _cstream_in_size(cs); }, outSize() { return _cstream_out_size(cs); } }; } /** * Create a push-based, single-threaded decompression stream. * @param {object} [opts] * @param {boolean} [opts.checksum=false] * @returns {{ decompress(Uint8Array): Uint8Array, finished(): boolean, free(): void, inSize(): number, outSize(): number }} */ function createDStream(opts) { const checksum = (opts && opts.checksum) || false; const optsPtr = _writeDecompressOpts(checksum); const ds = _dstream_create(optsPtr); _free(optsPtr); if (ds === 0) throw new Error('ZXC: failed to create dstream'); const inDescPtr = _malloc(IO_BUF_SIZE); const outDescPtr = _malloc(IO_BUF_SIZE); const stageCap = Math.max(_dstream_out_size(ds), 4096); const stagePtr = _malloc(stageCap); return { /** Push compressed bytes; return any decompressed bytes produced. */ decompress(data) { const srcPtr = data.length > 0 ? _malloc(data.length) : 0; try { if (srcPtr) Module.HEAPU8.set(data, srcPtr); _writeInbuf(inDescPtr, srcPtr, data.length); const chunks = []; let total = 0; let exhausted = data.length === 0; for (;;) { const beforeIn = _readPos(inDescPtr); _writeOutbuf(outDescPtr, stagePtr, stageCap); const r = _dstream_decompress(ds, outDescPtr, inDescPtr); if (r < 0) { throw new Error(`ZXC dstream error: ${_error_name(r)} (${r})`); } const produced = _readPos(outDescPtr); if (produced > 0) { chunks.push(new Uint8Array(Module.HEAPU8.buffer, stagePtr, produced).slice()); total += produced; } const afterIn = _readPos(inDescPtr); // Stop only when the call made no progress at all // (no input consumed AND no output produced). if (afterIn === beforeIn && produced === 0) break; if (!exhausted && afterIn === data.length) { _writeInbuf(inDescPtr, 0, 0); exhausted = true; } } return _concatChunks(chunks, total); } finally { if (srcPtr) _free(srcPtr); } }, /** True iff the file footer has been consumed and validated. */ finished() { return _dstream_finished(ds) !== 0; }, free() { _dstream_free(ds); _free(inDescPtr); _free(outDescPtr); _free(stagePtr); }, inSize() { return _dstream_in_size(ds); }, outSize() { return _dstream_out_size(ds); } }; } // --- WHATWG TransformStream adapters ------------------------------------- // // Mirror of the wrappers shipped by the Go, Rust, Python and Node bindings: // turn a CStream / DStream pair into a `TransformStream` so ZXC can be // wired into fetch pipelines, Service Workers, Deno, Bun, Node ≥18 — any // host that exposes the Streams API. Useful for OCI use cases such as // streaming a compressed layer through `fetch().body.pipeThrough(...)`. /** * Returns a WHATWG TransformStream that compresses bytes through a ZXC * frame. Pipe a ReadableStream through it to produce a * compressed ReadableStream. * * @param {object} [opts] * @param {number} [opts.level] * @param {boolean} [opts.checksum] * @returns {TransformStream} */ function createCompressTransformStream(opts) { let cs; return new TransformStream({ start() { cs = createCStream(opts); }, transform(chunk, controller) { try { const out = cs.compress(chunk); if (out.length > 0) controller.enqueue(out); } catch (e) { controller.error(e); try { cs.free(); } catch (_) { /* idempotent */ } } }, flush(controller) { try { const tail = cs.end(); if (tail.length > 0) controller.enqueue(tail); } catch (e) { controller.error(e); } finally { try { cs.free(); } catch (_) { /* idempotent */ } } }, }); } /** * Returns a WHATWG TransformStream that decompresses a ZXC frame. * * Errors the stream with a `'ZXC_TRUNCATED'`-tagged Error if the input * ends before the footer is reached. * * @param {object} [opts] * @param {boolean} [opts.checksum] * @returns {TransformStream} */ function createDecompressTransformStream(opts) { let ds; return new TransformStream({ start() { ds = createDStream(opts); }, transform(chunk, controller) { try { const out = ds.decompress(chunk); if (out.length > 0) controller.enqueue(out); } catch (e) { controller.error(e); try { ds.free(); } catch (_) { /* idempotent */ } } }, flush(controller) { try { if (!ds.finished()) { const err = new Error( 'ZXC: input drained before footer (truncated frame)'); err.code = 'ZXC_TRUNCATED'; controller.error(err); return; } } catch (e) { controller.error(e); } finally { try { ds.free(); } catch (_) { /* idempotent */ } } }, }); } // --- Exposed API object -------------------------------------------------- return Object.freeze({ compress, decompress, compressBound, getDecompressedSize, createCompressContext, createDecompressContext, createCStream, createDStream, createCompressTransformStream, createDecompressTransformStream, detectZxc, /** Library version string (e.g. "0.11.0"). */ version: _version_string(), /** Minimum compression level. */ minLevel: _min_level(), /** Maximum compression level. */ maxLevel: _max_level(), /** Default compression level. */ defaultLevel: _default_level(), /** Raw Emscripten Module (for advanced use). */ _module: Module, }); } zxc-0.11.0/zxcConfig.cmake.in000066400000000000000000000002431520102567100157340ustar00rootroot00000000000000@PACKAGE_INIT@ include(CMakeFindDependencyMacro) find_dependency(Threads) include("${CMAKE_CURRENT_LIST_DIR}/zxc-targets.cmake") check_required_components(zxc)